framework,version,device,op_name,kernel_source,gemm_dtype,m,n,k,latency
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,12288,40.060595703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,16384,54.78846028645834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,12288,74.30472005208334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,16384,65.57273356119791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,16384,99.03267415364584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,12288,52.502298990885414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,10240,33.25869954427083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,10240,62.18094482421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,8192,26.991609700520833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,10240,44.83531494140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,51200,167.926416015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,8192,49.68268636067708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,7168,23.355265299479164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,8192,33.427398681640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,7168,43.98326822916667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,7168,28.84490966796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,6144,20.34773152669271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,5120,17.12728271484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,6144,37.15346272786458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,5120,31.213330078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,6144,25.085209147135416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,4096,13.953328450520834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,4096,25.151920572916666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,5120,21.861124674479164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,3584,11.970799763997395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,3072,11.109731038411457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,3584,21.85755615234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,4096,17.262091064453124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,3072,18.911861165364584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,2560,9.062424723307291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,3584,14.897408040364585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,3072,12.591691080729166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,2560,16.563451131184898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,2048,7.308094787597656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,2048,12.75289306640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,2560,10.471531168619792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,1536,5.55118408203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,1024,4.808829752604167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,1536,9.419412231445312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,1024,6.798054504394531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,2048,9.212696329752605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,768,3.3027008056640623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,768,5.061389668782552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,512,2.7063913981119794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,512,3.6801333109537757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,1536,7.188517252604167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,256,2.2797035217285155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,256,1.8911829630533856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,128,1.7501611073811847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,128,1.64965337117513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,1024,5.363962809244791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,64,1.6247178395589192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,64,1.5901023864746093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,768,4.601566060384115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,32,1.5739210764567058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,512,3.544372304280599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,65536,32,1.546176020304362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,51200,254.14117838541665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,256,2.9248448689778646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,65536,128,2.7368095397949217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,65536,51200,320.73701171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,16384,77.92899576822916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,16384,44.157771809895834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,51200,131.40442708333333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,16384,51.88489990234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,12288,60.17491048177084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,12288,31.6605224609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,65536,173.16009114583332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,12288,39.86158040364583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,10240,26.426796468098956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,10240,49.820125325520834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,51200,197.43974609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,8192,21.198876953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,10240,34.93247477213542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,8192,39.22194010416667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,7168,18.591914876302084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,8192,26.49488932291667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,7168,34.05881958007812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,51200,244.10398763020834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,7168,23.658990478515626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,6144,16.04468790690104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,65536,247.86087239583335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,5120,13.284565226236978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,6144,29.43463541666667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,6144,20.778824869791666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,5120,24.427978515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,4096,10.856216430664062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,4096,20.215242513020833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,5120,17.076375325520836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,3584,10.331043497721355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,3584,17.460430908203126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,3072,8.46405741373698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,4096,12.452840169270832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,3072,14.583329264322916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,2560,7.132025655110676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,2560,12.493147786458334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,3584,11.038431803385418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,2048,5.774659220377604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,2048,9.70892333984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,1536,5.056305948893229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,3072,9.722128295898438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,1536,7.758880106608073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,2560,8.276156616210937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,1024,3.3958198547363283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,1024,5.323557535807291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,768,3.969911448160807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,768,2.8838422139485678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,512,2.884325408935547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,512,2.309099833170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,2048,7.121713256835937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,256,1.6896458943684896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,256,1.4278143564860026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,128,1.3889439900716147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,1536,5.6314142862955725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,128,1.314532216389974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,64,1.2909290313720703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,64,1.2674154917399088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,32,1.252234649658203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,51200,32,1.2331775665283202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,768,3.5936075846354165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,1024,4.33814697265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,512,2.7889249165852865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,256,2.3021578470865887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,51200,128,2.1757664998372395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,51200,65536,313.04020182291663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,16384,14.654713948567709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,16384,24.619814046223958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,12288,18.38848876953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,51200,46.11612141927083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,16384,17.67976277669271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,12288,10.50762227376302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,65536,58.19078776041666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,10240,16.257337443033855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,10240,9.363819376627605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,12288,12.155303955078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,51200,62.230122884114586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,8192,7.232743326822917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,8192,12.644601440429687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,10240,10.512959798177082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,51200,78.9588623046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,7168,6.62252705891927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,7168,10.771577962239583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,6144,9.3340576171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,6144,5.743390909830729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,8192,8.0589599609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,65536,79.12704264322916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,5120,7.9658655802408855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,5120,4.5080713907877605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,7168,6.974453226725261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,4096,3.7407168070475256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,4096,6.872560119628906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,3584,3.248762766520182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,3584,5.971952819824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,6144,6.133634948730469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,65536,98.27808430989583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,3072,3.050952657063802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,3072,4.726442464192709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,5120,5.154544067382813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,2560,2.3728363037109377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,2560,3.7983627319335938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,2048,1.9635061899820965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,2048,3.0873087565104167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,1536,1.5369781494140624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,4096,4.180547078450521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,1536,2.4434794108072917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,1024,1.0543349583943686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,1024,1.6441258748372394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,768,0.8749151865641276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,3584,3.748509979248047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,768,1.25688107808431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,512,0.9031487782796225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,512,0.6178709030151367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,2560,2.770952606201172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,256,0.5074645360310872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,256,0.4793162663777669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,128,0.435863463083903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,128,0.43329601287841796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,3072,3.3607297261555993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,64,0.4192917188008626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,64,0.4142143885294597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,2048,2.33132807413737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,16384,32,0.39444052378336586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,16384,32,0.40139414469401047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,1536,1.928430938720703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,1024,1.2889429728190103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,512,0.9188490549723307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,256,0.7828458786010742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,768,1.1833162943522135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,16384,128,0.715559450785319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,16384,10.823974609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,16384,19.786309814453126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,12288,15.003092447916666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,16384,12.061023966471355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,51200,35.23048909505208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,65536,45.07425537109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,12288,8.505365498860677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,10240,11.871898396809895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,10240,6.94119364420573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,51200,45.16112060546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,12288,9.239656575520833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,8192,5.834009806315104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,8192,9.142165120442709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,51200,56.743953450520834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,10240,8.184365844726562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,7168,5.144764709472656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,6144,4.403748067220052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,7168,7.977085876464844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,6144,6.997296142578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,65536,58.190861002604166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,5120,3.532330576578776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,8192,6.1157374064127605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,5120,6.077869669596354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,4096,2.940999348958333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,65536,73.66806640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,7168,5.460242207845052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,4096,4.9036310831705725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,3584,2.5882314046223955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,3584,4.06555404663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,3072,2.2263348897298174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,3072,3.409638468424479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,2560,1.8524650573730468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,2560,2.945659637451172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,6144,4.665403747558594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,5120,3.8686782836914064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,2048,2.3707178751627604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,2048,1.5204373677571614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,1536,1.724405288696289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,1536,1.1527008056640624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,1024,1.1977194468180339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,1024,0.8404597600301107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,4096,3.2152064005533854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,768,1.0187626520792643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,768,0.6696287790934246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,512,0.6378016153971354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,512,0.5029247919718425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,3584,2.923453776041667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,256,0.39160534540812175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,2560,2.1601397196451826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,256,0.3651381174723307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,128,0.3239679972330729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,128,0.3291562716166178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,3072,2.5529268900553386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,64,0.30574401219685876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,64,0.31759252548217776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,2048,1.758233642578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,12288,32,0.2984394709269206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,12288,32,0.3084320068359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,1536,1.3993333180745444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,768,0.874508794148763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,512,0.7120266596476237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,256,0.5925557454427083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,128,0.5440426508585612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,12288,1024,1.0329280217488608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,16384,9.350771077473958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,16384,16.175664265950523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,12288,11.287916056315105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,51200,30.429758707682293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,16384,10.115462239583334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,12288,7.304810587565105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,65536,38.88951416015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,10240,5.966153462727865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,10240,9.565437825520833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,51200,35.38064778645833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,12288,7.724045817057292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,8192,4.845513407389323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,51200,49.5571044921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,8192,7.994622802734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,10240,6.705079650878906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,7168,4.634735107421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,6144,3.6318206787109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,7168,7.427029418945312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,6144,5.651444498697916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,65536,61.2872314453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,5120,3.1274335225423178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,65536,53.207096354166666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,5120,5.609089152018229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,8192,5.16871337890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,4096,2.527060190836589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,3584,2.135214996337891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,4096,3.816051228841146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,7168,4.71078847249349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,3584,3.3904159545898436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,3072,1.823736572265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,3072,2.8377408345540363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,2560,2.4709846496582033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,2560,1.5732672373453775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,5120,3.3416788736979166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,6144,3.836991882324219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,2048,1.326818084716797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,2048,1.925702412923177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,1536,1.4455626169840494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,1536,1.045461336771647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,1024,0.7399914423624675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,1024,0.996887461344401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,768,0.7757045110066731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,768,0.5775338490804036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,4096,2.856159973144531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,512,0.5563338597615559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,512,0.4625631968180339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,3584,2.490010579427083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,2560,1.765435791015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,256,0.37189973195393883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,3072,2.101806894938151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,256,0.3163669268290202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,128,0.28448108037312825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,128,0.2811637242635091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,2048,1.4627360026041667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,64,0.26705919901529945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,64,0.2683616002400716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,10240,32,0.24701333045959473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,10240,32,0.2591104030609131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,1536,1.1058719635009766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,768,0.734060796101888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,1024,0.8259967803955078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,512,0.6047519683837891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,256,0.5008970578511556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,10240,128,0.4542613347371419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,16384,8.038748677571615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,16384,12.066078694661458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,12288,9.188682047526042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,51200,25.342281087239584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,16384,8.283111572265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,12288,6.317622375488281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,65536,33.62439778645833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,10240,5.20901845296224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,10240,8.9749267578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,51200,29.86925048828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,51200,39.98386637369792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,8192,4.053734334309896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,12288,6.238629150390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,8192,6.263582865397135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,7168,3.5984725952148438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,10240,5.436850992838542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,7168,5.913615926106771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,6144,5.13565419514974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,65536,40.02901611328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,6144,3.1524447123209636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,5120,2.6859840393066405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,5120,3.924309285481771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,65536,52.24942220052084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,4096,3.4155871073404946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,4096,2.0977962493896483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,3584,2.846901448567708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,7168,3.7119135538736976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,8192,4.350191752115885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,3584,1.8397824605305988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,3072,1.5567498524983725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,2560,2.010742441813151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,3072,2.483276875813802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,2560,1.3259722391764321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,2048,1.6914666493733723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,2048,1.0765300750732423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,6144,3.3266143798828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,5120,2.7290603637695314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,1536,1.2099039713541666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,1536,0.8534250895182292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,1024,0.8631818771362305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,1024,0.6141429265340169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,768,0.6222346623738606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,768,0.5135840098063151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,4096,2.203227742513021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,512,0.4759978612263997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,512,0.3642357190450033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,3584,1.89583257039388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,256,0.2844501177469889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,256,0.2681269327799479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,2560,1.4536309560139975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,128,0.22373973528544105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,128,0.23450026512145997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,3072,1.740170669555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,64,0.20968960126241049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,64,0.21860693295796713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,8192,32,0.19698239962259928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,8192,32,0.21671679814656578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,2048,1.1803092956542969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,1536,0.9033599853515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,512,0.49122772216796873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,768,0.600985590616862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,256,0.40563414891560867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,1024,0.6704383850097656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,8192,128,0.3711317380269369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,16384,7.124554443359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,16384,10.266208902994792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,12288,8.299960327148437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,51200,23.662654622395834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,12288,5.32047373453776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,16384,7.404857381184895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,65536,29.729457600911456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,10240,4.525704447428385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,10240,7.35177510579427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,51200,34.44270833333333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,51200,26.527933756510418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,8192,3.657897694905599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,8192,5.5796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,12288,5.712567647298178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,7168,5.225282287597656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,7168,3.2130218505859376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,10240,4.710328674316406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,65536,32.062306722005204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,65536,45.3697265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,6144,3.998040517171224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,6144,2.8865386962890627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,5120,2.2245407104492188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,5120,3.3348203023274743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,4096,1.8455968221028645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,4096,2.7222078959147136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,3584,1.6168927510579425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,3584,2.4722496032714845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,7168,3.3858751932779945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,8192,3.97754872639974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,3072,1.9819498697916669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,3072,1.375176493326823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,2560,1.1669568379720052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,2560,1.691109339396159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,2048,1.0243669509887696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,2048,1.422213363647461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,5120,2.466599527994792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,6144,2.924468231201172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,1536,1.1845834096272787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,1536,0.7810890833536784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,1024,0.5451050440470377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,1024,0.7416842778523762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,4096,1.968172836303711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,768,0.5737141291300456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,768,0.44978240331013997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,512,0.37503039042154945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,512,0.32310078938802084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,3584,1.6443274180094403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,256,0.24841599464416503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,2560,1.292123794555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,256,0.23511039415995277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,128,0.1990005334218343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,128,0.20548586845397948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,3072,1.4523167928059897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,64,0.18699839909871419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,64,0.19710826873779297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,7168,32,0.17370559374491373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,7168,32,0.188809601465861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,2048,1.0361109415690104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,1536,0.8082784016927084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,512,0.4457088152567546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,768,0.5466410954793294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,1024,0.6157930374145508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,256,0.36391468048095704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,7168,128,0.32889814376831056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,16384,6.394132486979166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,16384,8.863621012369792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,12288,7.973753356933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,51200,20.7618408203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,12288,4.780168660481771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,16384,6.44951171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,65536,27.036863199869792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,51200,28.378306070963543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,10240,5.573148600260416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,10240,4.069897715250651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,51200,22.443768310546876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,8192,3.5279894510904946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,8192,4.645875040690104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,12288,4.881959533691406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,7168,3.9249855041503907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,65536,36.96649169921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,7168,2.897360992431641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,6144,2.4966763814290363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,10240,4.316070556640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,6144,3.291097513834635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,5120,2.837951914469401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,65536,31.215321858723957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,5120,2.227099609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,4096,2.2972043355305987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,4096,1.660505549112956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,3584,1.4119883219401042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,3584,2.1727254231770834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,8192,3.3976959228515624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,3072,2.1556500752766925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,3072,1.2546143849690756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,2560,1.055190404256185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,7168,2.9748863220214843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,2560,1.586628214518229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,2048,1.1603338877360025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,2048,0.8889365514119467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,1536,0.8749472300211588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,5120,2.2187850952148436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,6144,2.52272211710612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,1536,0.6898517608642578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,1024,0.6005631764729817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,1024,0.47639039357503254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,768,0.43704961140950516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,768,0.38661225636800134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,4096,1.683018620808919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,512,0.3416810671488444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,512,0.2963456153869629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,3584,1.4110827128092447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,256,0.21298453013102212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,256,0.2030229409535726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,2560,1.055563735961914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,128,0.15935786565144855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,128,0.17660800615946454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,3072,1.3009813944498698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,64,0.14982612927754718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,2048,0.8720874786376953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,64,0.17029226620992025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,6144,32,0.14688746134440106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,6144,32,0.1615839958190918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,1536,0.713645871480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,512,0.38138132095336913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,768,0.46043945948282883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,1024,0.525160535176595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,128,0.28436692555745446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,6144,256,0.3105951944986979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,16384,7.591469828287761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,16384,5.643162536621094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,12288,5.669337463378906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,51200,18.447328694661458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,12288,4.781493123372396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,65536,24.08375040690104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,16384,5.504316711425782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,51200,25.60697021484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,10240,4.8577631632486975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,10240,3.751274617513021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,51200,19.058121744791666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,8192,2.901866658528646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,8192,4.293761189778646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,65536,32.22643839518229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,7168,3.3960566202799476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,7168,2.502099100748698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,65536,23.461968994140626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,12288,4.371190388997396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,6144,2.9093663533528646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,6144,2.1389259338378905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,5120,1.7787178039550782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,5120,2.3903284708658856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,4096,2.0464202880859377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,4096,1.4499380747477213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,10240,3.6378634134928385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,3584,1.2728628794352213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,3584,1.8877323150634766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,3072,1.4881248474121094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,3072,1.1780138651529948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,2560,1.375431442260742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,7168,2.529127502441406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,8192,3.0203926086425783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,2560,0.9475029627482096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,2048,1.008138656616211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,2048,0.7988608042399089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,1536,0.7282581329345703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,5120,1.8153770446777344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,6144,2.17390619913737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,1536,0.6067626953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,1024,0.5310837427775066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,1024,0.42534720102945967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,768,0.37749226888020837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,768,0.3569098790486654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,4096,1.3810133616129556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,512,0.29188267389933265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,512,0.2785344123840332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,3584,1.22477658589681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,256,0.1667402744293213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,256,0.18098986943562825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,2560,0.9132319768269858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,128,0.14285227457682292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,128,0.1572757403055827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,3072,1.092407480875651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,64,0.1335349400838216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,64,0.14898239771525065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,5120,32,0.12523626486460368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,5120,32,0.14216213226318358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,2048,0.7728064219156902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,1536,0.6069194793701171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,512,0.33068161010742186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,768,0.40661013921101885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,256,0.267796262105306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,1024,0.45513067245483396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,5120,128,0.2400042692820231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,16384,4.873832702636719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,16384,5.99375254313151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,12288,4.540794881184896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,12288,4.218561045328776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,51200,15.877560424804688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,65536,20.42015177408854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,51200,20.177630615234374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,10240,3.99977060953776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,16384,4.652777608235677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,10240,3.160857645670573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,65536,24.681221516927085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,8192,3.1472447713216143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,8192,2.5103050231933595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,51200,16.045902506510416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,7168,3.0360491434733072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,7168,2.165593719482422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,6144,2.3331273396809893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,6144,1.952461878458659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,12288,3.6513771057128905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,5120,1.9514624277750652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,5120,1.5581204732259115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,65536,21.390494791666665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,10240,2.937316385904948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,4096,1.5646165211995444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,4096,1.2555296579996744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,3584,1.0983423868815103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,3584,1.2961631774902345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,3072,1.2005067189534506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,3072,0.9856255849202474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,7168,2.1120021820068358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,8192,2.478595225016276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,2560,1.0051968256632486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,2560,0.8248170852661133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,2048,0.7424458821614583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,6144,1.7383968353271484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,5120,1.4605803171793619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,2048,0.6979647954305013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,1536,0.5953813552856445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,1536,0.5425770441691081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,1024,0.4266976038614909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,1024,0.3698314666748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,768,0.2980682690938314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,768,0.30015252431233724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,4096,1.1708821614583333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,512,0.23647359212239585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,512,0.24112426439921059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,3584,1.0354026794433593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,256,0.135641606648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,256,0.15063145955403645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,2560,0.7500362396240234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,128,0.10915839672088623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,128,0.12506346702575682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,3072,0.8679882685343424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,64,0.10172479947408039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,64,0.12090026537577311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,4096,32,0.10141759713490803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,2048,0.6252074559529622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,4096,32,0.11527466773986816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,1536,0.5075039863586426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,768,0.3269098599751791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,256,0.21525333722432455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,512,0.26566826502482094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,1024,0.37339626948038734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,4096,128,0.19251519838968914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,16384,4.480216471354167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,16384,5.258837381998697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,12288,3.978690083821615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,51200,14.109308878580729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,12288,3.5366198221842446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,51200,17.886045328776042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,65536,18.80644327799479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,10240,3.510975901285807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,65536,22.65410359700521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,10240,2.8168820699055988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,16384,4.294771321614584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,51200,14.179180908203126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,8192,2.2475850423177084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,7168,2.4279604593912762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,8192,2.6597803751627604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,7168,2.1479263305664062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,6144,1.9888863881429035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,6144,1.6947840372721354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,12288,3.1893600463867187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,5120,1.4348480224609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,5120,1.6588778177897137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,65536,19.439306640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,10240,2.7069046020507814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,4096,1.1627370198567708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,4096,1.3371242523193358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,3584,1.0118560155232748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,3584,1.1424149831136068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,3072,0.9028810501098633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,3072,1.1844202677408853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,2560,0.806602668762207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,8192,2.120840581258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,7168,1.8359914143880207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,2560,0.800108782450358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,2048,0.6672725041707357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,2048,0.6215765635172527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,1536,0.5038773218790691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,5120,1.2897727966308594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,1536,0.5039615948994954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,1024,0.3390133221944173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,6144,1.5959498087565103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,1024,0.36127894719441733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,768,0.2664426644643148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,768,0.278443717956543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,4096,1.0777973175048827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,512,0.21655786832173668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,512,0.21997973124186196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,3584,0.9565813064575195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,256,0.1284874677658081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,256,0.14151040712992352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,2560,0.6692330678304036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,128,0.10232853094736735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,128,0.1151968002319336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,3072,0.8069973627726237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,64,0.08920959631601968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,64,0.10769813060760498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3584,32,0.08819626967112223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3584,32,0.10378133455912272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,2048,0.5623999913533528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,1536,0.45395199457804364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,256,0.19448107083638508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,512,0.24240320523579917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,768,0.2993397394816081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,128,0.1746239980061849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3584,1024,0.34324372609456383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,16384,4.556699625651041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,16384,4.069586181640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,12288,3.297520955403646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,51200,13.146342976888022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,65536,16.307450358072916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,51200,14.657613118489584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,65536,19.184977213541664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,12288,3.045514678955078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,10240,2.8755147298177084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,10240,2.656205749511719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,8192,2.424420166015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,8192,2.0549386342366533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,51200,11.661836751302083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,7168,2.229516855875651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,16384,3.8375712076822914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,7168,1.787986119588216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,6144,1.747303517659505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,6144,1.5606048583984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,5120,1.4311135609944663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,65536,15.965597534179688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,12288,2.9106730143229167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,5120,1.3035733540852865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,4096,1.117297108968099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,10240,2.3541770935058595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,4096,1.0759754180908203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,3584,1.0033077239990233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,3584,0.9406218846638998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,3072,0.8805536270141602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,3072,0.813748296101888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,2560,0.7188575744628907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,2560,0.6988373438517252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,8192,1.87850341796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,7168,1.610165278116862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,2048,0.5908479690551758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,2048,0.5717290878295899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,1536,0.40398505528767903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,1536,0.44746987024943036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,5120,1.1178143819173179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,6144,1.3602891286214194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,1024,0.2993642807006836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,1024,0.3171039899190267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,768,0.21736000378926595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,768,0.24989333152770996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,4096,0.9106325149536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,512,0.15943360328674316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,512,0.19495147069295246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,3584,0.8502538681030274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,256,0.11514453093210857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,256,0.12958506743113202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,2560,0.5969226837158204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,128,0.08419626553853353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,128,0.10470293362935383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,3072,0.7073834737141926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,64,0.07903146743774414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,64,0.09909013112386068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,3072,32,0.07741119861602783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,3072,32,0.09435946941375732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,2048,0.5028842608133952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,1536,0.40209172566731766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,768,0.26340160369873045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,512,0.2125482718149821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,1024,0.30063254038492837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,256,0.16907839775085448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,3072,128,0.15131840705871583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,16384,3.748209126790365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,16384,3.699235280354818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,12288,2.788159942626953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,12288,2.79108149210612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,51200,11.342466227213542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,65536,16.15989481608073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,51200,11.797699991861979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,65536,15.147923787434896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,10240,2.7324757893880207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,8192,1.9395871480305988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,10240,2.3347381591796874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,8192,1.9733600616455078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,7168,1.6725802103678387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,7168,1.650595219930013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,16384,3.4146410624186196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,51200,10.375447591145832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,6144,1.465866724650065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,6144,1.4587263743082681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,5120,1.2141727447509765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,5120,1.2081940968831382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,65536,13.757980346679688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,4096,0.9184384028116861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,10240,2.027512486775716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,12288,2.4832778930664063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,3584,0.8520640055338541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,4096,1.1296490987141927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,3584,0.8557674407958984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,3072,0.7111701329549154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,3072,0.7295007705688477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,2560,0.5656490961710612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,2560,0.6415189107259114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,7168,1.4271828969319662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,2048,0.4854602813720703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,8192,1.6866111755371094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,2048,0.5257184028625488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,5120,0.9755082448323569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,1536,0.365717347462972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,6144,1.1970837910970054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,1536,0.4148970603942871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,1024,0.24199466705322265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,1024,0.29066880544026696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,768,0.18817280133565267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,4096,0.8013770421346029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,768,0.22993599573771156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,512,0.13920000394185383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,512,0.17320960362752277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,3584,0.7162879943847656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,256,0.09462292989095053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,2560,0.5195914586385091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,256,0.11274560292561848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,128,0.0724501371383667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,128,0.09304853280385336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,3072,0.6181088129679362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,64,0.06778346697489421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,64,0.08601600329081217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2560,32,0.06730559666951498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2560,32,0.08202026685078939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,2048,0.4391818682352702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,128,0.1278165340423584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,1536,0.35104640324910485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,512,0.1818677266438802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,256,0.1438762664794922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,1024,0.2645034631093343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2560,768,0.2283616065979004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,16384,2.906541951497396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,16384,3.421395111083984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,12288,2.3675882975260416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,65536,11.966825358072917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,51200,9.137788899739583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,12288,2.5794293721516928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,65536,13.772993977864584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,51200,10.670774332682292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,10240,1.833073043823242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,10240,2.1474965413411455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,8192,1.5226708730061849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,8192,1.8183563232421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,7168,1.3587467193603515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,7168,1.6308096567789714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,6144,1.2292383829752604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,16384,2.762828826904297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,6144,1.2958175659179687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,51200,9.102508544921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,12288,2.0694911956787108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,5120,0.9315381368001303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,5120,1.084221903483073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,4096,0.789689572652181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,4096,0.8598005294799804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,10240,1.699668248494466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,3584,0.6319477081298828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,65536,11.742562866210937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,3584,0.7699093500773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,3072,0.5793312072753907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,3072,0.6591989517211914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,2560,0.4387392044067383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,2560,0.5748106638590496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,7168,1.1931605021158853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,2048,0.3827583948771159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,8192,1.4196127573649089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,2048,0.487885856628418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,1536,0.2715840021769206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,1536,0.36352532704671225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,5120,0.8343114852905273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,1024,0.20887039502461752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,1024,0.27728853225708006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,6144,0.9967541376749673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,768,0.15414613087972004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,768,0.2100213368733724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,4096,0.6796127955118816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,512,0.10933972994486492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,512,0.15101866722106932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,2560,0.44547306696573896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,256,0.06854720115661621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,3584,0.5974602381388346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,256,0.0995690663655599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,128,0.05916800101598104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,128,0.0807861328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,3072,0.5217642784118652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,64,0.0549941341082255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,64,0.07641812960306803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,2048,32,0.0544543981552124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,2048,32,0.07172266642252603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,2048,0.37770134607950845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,1536,0.3004053433736166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,512,0.15245973269144694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,768,0.19035414059956868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,128,0.10233173370361329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,256,0.1184928019841512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,2048,1024,0.22204693158467612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,16384,2.3696693420410155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,16384,3.0143231709798175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,65536,9.045066324869792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,12288,1.794988759358724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,51200,7.003892517089843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,12288,2.2684811909993488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,10240,1.4071114857991538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,65536,12.31815694173177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,51200,9.424839274088542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,10240,1.9294453938802083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,8192,1.0719402313232422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,8192,1.5349194844563803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,7168,1.046941884358724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,7168,1.3348309834798178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,6144,0.8727306365966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,51200,7.569913736979167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,6144,1.1655957539876303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,12288,1.6607701619466144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,5120,0.7083882649739583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,16384,2.3037109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,65536,9.60402119954427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,4096,0.5659114837646484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,5120,0.9764821370442709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,3584,0.46755094528198243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,4096,0.8007754643758138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,10240,1.4505760192871093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,3584,0.698862902323405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,3072,0.43528426488240557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,3072,0.5962474822998047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,2560,0.33978239695231116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,2560,0.5047520001729329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,8192,1.083502960205078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,7168,0.9800512313842773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,2048,0.2804106712341309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,2048,0.4192149480183919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,1536,0.21413440704345704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,1536,0.32248427073160807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,5120,0.7032042821248372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,1024,0.15072959264119465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,1024,0.23274025917053223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,6144,0.8382549285888672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,768,0.11736213366190593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,768,0.18278506596883137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,4096,0.5729504267374674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,512,0.0841813325881958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,512,0.13359999656677246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,2560,0.37592748006184895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,3584,0.5096405347188313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,256,0.052553598086039224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,256,0.08571733633677164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,128,0.04705599943796794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,128,0.06745813687642416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,3072,0.44718612035115557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,64,0.04291093349456787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,64,0.06204266548156738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1536,32,0.043186132113138834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1536,32,0.059249067306518556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,2048,0.31309439341227213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,1536,0.25153706868489584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,128,0.08326720396677653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,256,0.09549333254496256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,512,0.1251477320988973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,768,0.15720960299173992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1536,1024,0.18641386032104493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,65536,5.9714411417643225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,16384,1.5269643147786458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,12288,1.1394378662109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,16384,2.6507776896158854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,51200,4.685723876953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,10240,0.9224063873291015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,12288,1.9853312174479167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,10240,1.6624980926513673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,8192,0.7179733276367187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,65536,10.676168823242188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,51200,8.336460876464844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,7168,0.606218655904134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,8192,1.3494463602701823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,7168,1.1887498219807944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,6144,0.5155296007792155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,6144,1.02227840423584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,16384,1.8739636739095051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,51200,5.892757161458333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,12288,1.3315797170003256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,5120,0.44713706970214845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,10240,1.1167711893717447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,4096,0.36014188130696617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,5120,0.8541354497273763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,3584,0.305948797861735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,4096,0.6957152048746745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,65536,7.559349568684896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,3584,0.607592519124349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,3072,0.28094825744628904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,3072,0.5217877388000488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,2560,0.2230303923288981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,2560,0.4415370623270671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,7168,0.7817770640055339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,2048,0.18530559539794922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,2048,0.37435092926025393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,8192,0.8720309575398764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,1536,0.13948906262715657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,1536,0.28681281407674153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,5120,0.5561930974324544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,1024,0.09791893164316813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,6144,0.6669727961222331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,1024,0.1974453290303548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,768,0.07748586336771647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,768,0.15760746002197265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,4096,0.4624938646952311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,2560,0.30032958984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,512,0.05869973500569662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,3584,0.4069674809773763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,512,0.11496106783548991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,256,0.038540800412495926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,256,0.07261013189951579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,128,0.03346133232116699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,128,0.054347733656565346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,3072,0.35531946818033855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,64,0.030357333024342854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,64,0.0499669353167216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,1024,32,0.03104746739069621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,1024,32,0.047414398193359374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,2048,0.2512394587198893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,1536,0.19813332557678223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,256,0.07054933706919352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,65536,4.487602233886719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,128,0.05990826686223348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,512,0.09381333192189535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,1024,0.14561492602030437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,1024,768,0.12169919808705647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,16384,1.1254868825276694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,12288,0.8711765289306641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,51200,3.6626485188802085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,16384,2.460376485188802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,10240,0.642414919535319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,12288,1.8503210703531903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,10240,1.564919408162435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,8192,0.516866143544515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,65536,10.017490641276042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,8192,1.25666135152181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,51200,7.799997965494792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,7168,0.4537109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,7168,1.095424016316732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,6144,0.3957056045532227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,6144,0.953548812866211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,16384,1.6192138671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,5120,0.33771947224934895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,51200,4.980731709798177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,65536,6.316682434082031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,10240,0.9650058746337891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,12288,1.1440896352132162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,4096,0.2654922644297282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,3584,0.2326752026875814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,5120,0.8045642852783204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,4096,0.6424032211303711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,3072,0.20303786595662437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,3584,0.5714485168457031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,3072,0.5035914738972982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,2560,0.17211413383483887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,2560,0.4156341234842936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,7168,0.6775722503662109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,2048,0.13887999852498373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,8192,0.7770730972290039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,2048,0.33621546427408855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,1536,0.10753493309020996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,1536,0.2594634691874186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,5120,0.48996585210164384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,1024,0.07546026706695556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,6144,0.5855018615722656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,1024,0.18420906066894532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,768,0.060003201166788735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,768,0.14670613606770833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,4096,0.4014464060465495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,512,0.04741866588592529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,2560,0.26290987332661947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,512,0.10678400198618572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,256,0.031880533695220946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,256,0.06466879844665527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,3584,0.360315736134847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,128,0.027508266766866046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,128,0.04813333352406819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,3072,0.30925013224283854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,64,0.02445333401362101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,64,0.04458453257878621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,768,32,0.025011199712753295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,768,32,0.04097493489583333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,2048,0.21907307306925455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,1536,0.17211626370747884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,65536,3.195538075764974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,128,0.04782613515853882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,256,0.05832320054372152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,512,0.0788640022277832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,768,0.10344746907552083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,51200,2.328174845377604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,16384,0.7216341018676757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,12288,0.5867434819539388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,768,1024,0.12445653279622396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,16384,2.2968724568684893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,10240,0.4351978619893392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,12288,1.7454378763834637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,10240,1.4568672180175781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,8192,0.3483978589375814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,65536,9.341131591796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,8192,1.1720811208089192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,7168,0.305948797861735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,51200,7.269979858398438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,7168,1.0281440099080403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,6144,0.26385706265767417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,51200,4.23474349975586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,12288,0.984446907043457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,10240,0.8447999954223633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,16384,1.3029034932454429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,5120,0.22515625953674318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,6144,0.8801429112752279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,4096,0.17953707377115885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,65536,5.444043986002604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,3584,0.16564052899678547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,5120,0.7369183858235677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,4096,0.5940010706583659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,3072,0.13789653778076172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,3584,0.5258240063985189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,3072,0.45157012939453123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,2560,0.11701546510060627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,2560,0.3809119860331217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,7168,0.5903296152750651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,2048,0.0979594628016154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,2048,0.3100384076436361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,8192,0.6705759684244792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,1536,0.07570026715596517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,1536,0.23930133183797203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,5120,0.42862294514973953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,1024,0.05396800041198731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,1024,0.17021759351094562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,4096,0.34643306732177737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,768,0.0431007981300354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,768,0.13553600311279296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,2560,0.2266613324483236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,512,0.03329600095748901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,6144,0.5083168029785157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,512,0.09842879772186279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,256,0.024819199244181314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,256,0.05743360122044881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,3584,0.3073002815246582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,128,0.018717867136001588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,128,0.041203200817108154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,3072,0.26826133728027346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,64,0.01513920029004415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,64,0.03540053367614746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,512,32,0.01560533344745636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,512,32,0.032866134246190386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,2048,0.18901440302530925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,65536,1.6261717478434246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,1536,0.14653973579406737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,128,0.03587520122528076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,51200,1.2765888214111327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,256,0.04564799865086873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,16384,0.4036266644795735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,512,0.06442559957504272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,768,0.0852288007736206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,12288,0.31031360626220705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,512,1024,0.10446826616923015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,16384,2.1608352661132812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,10240,0.25922560691833496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,12288,1.6263776143391926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,10240,1.3553333282470703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,8192,0.20797440210978188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,65536,8.68754374186198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,8192,1.0890186309814454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,7168,0.1856053352355957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,51200,6.75531005859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,7168,0.9562016169230143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,6144,0.16134293874104816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,6144,0.8237663904825846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,51200,3.5084330240885415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,5120,0.13536639213562013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,65536,4.505098470052084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,5120,0.6928885142008464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,12288,0.8479114532470703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,16384,1.108468246459961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,4096,0.11119039853413899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,3584,0.09880320231119791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,10240,0.7092437108357748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,4096,0.5554239908854167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,3584,0.4919967969258626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,3072,0.08688639799753825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,3072,0.42497812906901045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,2560,0.07411626974741617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,2560,0.3578954696655273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,7168,0.5097077369689942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,2048,0.06085120042165121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,2048,0.2884288152058919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,8192,0.5775871912638346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,1536,0.048264535268147786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,1536,0.2228991985321045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,6144,0.44075733820597335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,1024,0.034169598420461016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,1024,0.15572586059570312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,5120,0.37137066523234047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,768,0.028182399272918702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,768,0.12272853056589764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,2560,0.19421226183573406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,512,0.020933334032694498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,4096,0.2998378753662109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,512,0.0869856039683024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,256,0.011778133114178975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,256,0.047075200080871585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,3584,0.2650229295094808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,128,0.00939520001411438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,3072,0.2303167978922526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,128,0.029864533742268877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,2048,0.1568170706431071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,64,0.008005333443482716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,64,0.025272534290949507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,256,32,0.007985066870848339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,256,32,0.022730666399002075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,1536,0.12147413094838459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,65536,1.5061983744303384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,256,0.032842665910720825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,768,0.0669823964436849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,51200,1.185647964477539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,16384,0.3840832074483236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,512,0.049693866570790605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,128,0.023293866713841756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,12288,0.29393065770467125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,256,1024,0.08412586847941081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,16384,2.1355105082194012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,10240,0.2453546682993571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,12288,1.6100448608398437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,10240,1.348031997680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,8192,0.19767786661783854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,8192,1.0794357299804687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,51200,6.730231221516926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,7168,0.17316266695658367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,65536,8.655952962239583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,7168,0.9437941233317056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,6144,0.15177280108133953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,6144,0.813144556681315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,51200,3.3176063537597655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,5120,0.14706026713053386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,12288,0.8132202784220377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,5120,0.6829023996988932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,4096,0.10531413555145264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,10240,0.6805856068929036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,65536,4.32800547281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,3584,0.10483520030975342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,4096,0.5505098978678385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,16384,1.0741610209147134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,3072,0.07934292952219645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,3072,0.4160426775614421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,3584,0.48210560480753584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,2560,0.06772267023722331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,2560,0.34925225575764973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,7168,0.4859039942423503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,2048,0.0551797350247701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,2048,0.28375466664632165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,8192,0.5489440282185872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,1536,0.04408106803894043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,1536,0.21652372678120932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,6144,0.41747732162475587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,1024,0.03174186746279399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,5120,0.351751454671224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,1024,0.15000319480895996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,768,0.02563733259836833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,768,0.11570453643798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,4096,0.28368746439615883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,512,0.01323199967543284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,512,0.0801749308904012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,3584,0.2519050598144531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,256,0.00823466678460439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,2560,0.18323520024617512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,256,0.03970346848169963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,128,0.006570666531721751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,128,0.025064533948898314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,3072,0.21621440251668295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,64,0.005985066791375478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,64,0.021504000822703043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,128,32,0.006183466811974844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,128,32,0.02014933427174886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,2048,0.14745920499165852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,65536,1.4777076721191407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,1536,0.11232533454895019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,51200,1.1570442199707032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,16384,0.37584638595581055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,512,0.041960533459981283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,12288,0.28517653147379557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,128,0.0168287992477417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,768,0.05940586725870768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,256,0.024708267052968344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,10240,0.23801280657450358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,8192,0.19040533701578777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,7168,0.16831040382385254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,16384,2.121786626180013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32768,128,1024,0.07666880289713542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,6144,0.17065067291259767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,8192,1.0754645029703775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,5120,0.12251093387603759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,10240,1.3374677022298178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,12288,1.5995520273844401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,4096,0.10041066805521648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,3584,0.08770346641540527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,3072,0.0758453369140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,6144,0.8079701105753581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,4096,0.5456586837768554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,3584,0.4802463849385579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,5120,0.6798271814982096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,2560,0.06477013429005941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,2048,0.05220906734466553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,1536,0.04129386742909749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,3072,0.41244694391886394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,1024,0.029153066873550414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,2048,0.2817280133565267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,2560,0.34773333867390954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,768,0.023098667462666832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,1024,0.14708479245503742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,512,0.00853013296922048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,256,0.00614933321873347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,768,0.1129856030146281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,512,0.07620586554209391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,128,0.005167999863624572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,256,0.035545599460601804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,64,0.00469760000705719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,128,0.024182399113972984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,64,0.020499199628829956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,64,32,0.004705066482226053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,32,0.018167465925216675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,7168,0.9393610636393228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,1536,0.21399253209431968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,51200,1.1475327809651694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,51200,6.716764831542969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,65536,1.4704981486002604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,16384,0.3705397288004557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,12288,0.3058858553568522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,10240,0.23508799870808922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,8192,0.19162240028381347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,64,65536,8.638238525390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,7168,0.1952191988627116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,10240,1.3339978535970052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,8192,1.064520517985026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,12288,1.591762161254883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,6144,0.14438826243082684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,16384,2.1204415639241536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,5120,0.1207594633102417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,4096,0.0986303965250651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,3584,0.08592853546142579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,3072,0.0746122678120931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,7168,0.938530158996582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,5120,0.6744469324747722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,4096,0.5422805150349934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,6144,0.8029823939005534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,2560,0.06273599863052368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,3584,0.47695039113362625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,3072,0.4101397196451823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,2048,0.050785064697265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,1536,0.03975679874420166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,1024,0.027525333563486735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,768,0.02192853291829427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,512,0.007469866673151653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,1024,0.14567467371622722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,1536,0.2126688003540039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,256,0.005383466680844625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,512,0.07502079804738362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,2560,0.34516372680664065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,128,0.004394666850566864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,64,0.0038293334345022834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,256,0.03280533353487651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,128,0.02318613330523173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32768,32,32,0.0038805333276589714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,64,0.019451733430226645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,32,0.017489065726598106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,768,0.11136000156402588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,2048,0.27775999704996746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,51200,6.687147521972657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32768,32,65536,8.62317403157552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,12288,20.589536539713542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,16384,26.744716389973956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,16384,32.79471435546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,12288,38.01865234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,16384,50.54781494140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,12288,25.1515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,10240,17.454852294921874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,10240,31.241404215494793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,10240,23.29631551106771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,8192,13.105416870117187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,8192,17.41262003580729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,8192,25.01449178059896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,7168,12.906052652994791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,7168,22.091939290364586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,51200,82.57332356770834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,7168,14.724172973632813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,6144,9.926767985026043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,5120,8.638848876953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,6144,19.649357096354166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,6144,11.958792114257813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,5120,9.73377685546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,5120,15.926666259765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,4096,12.479529825846354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,4096,6.855222574869792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,3584,6.9611562093098955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,3072,5.230623881022135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,4096,7.96629638671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,3584,10.967545572916666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,3584,7.181532796223958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,3072,9.834290568033854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,2560,4.427196756998698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,2048,3.636556752522787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,2560,7.898121643066406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,3072,6.381449381510416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,2048,6.753613789876302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,2560,5.4130615234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,2048,4.889479573567709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,1536,2.843436686197917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,1536,4.855118815104166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,1024,1.9022655487060547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,1536,3.6112299601236977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,1024,3.287187194824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,768,1.616815948486328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,768,2.637156168619792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,512,1.2063563028971354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,512,1.8507349650065106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,256,1.1720235188802084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,256,0.9001696268717447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,128,0.8743178685506185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,1024,2.6555018107096355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,128,0.8328106562296549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,64,0.8157632191975912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,64,0.8053845087687174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,768,2.294420369466146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,32,0.7902506510416667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,65536,32,0.7836554845174153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,512,1.7903680165608722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,256,1.4917760213216147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,128,1.406008529663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,65536,51200,130.31942545572917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,65536,51200,154.70870768229167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,16384,39.07046305338542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,16384,20.762809244791665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,51200,67.76986897786459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,16384,25.10693359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,12288,29.03800048828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,65536,81.9914306640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,12288,16.154939778645833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,12288,19.664794921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,10240,12.87020060221354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,51200,93.51357421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,10240,24.386651611328126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,8192,10.52644755045573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,10240,18.598018391927084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,8192,19.667041015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,8192,12.577088419596354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,7168,9.281307983398438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,7168,10.978966267903646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,7168,17.091058349609376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,6144,7.7920491536458325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,51200,122.2081298828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,6144,14.929000854492188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,6144,8.964186604817709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,5120,6.746631368001301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,65536,127.1181640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,4096,9.853189086914062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,4096,5.441700236002604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,5120,12.047920735677083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,5120,7.858729553222656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,3584,4.848856608072916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,4096,6.1961003621419275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,3072,4.224189758300781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,3584,8.724273681640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,3072,7.644939676920574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,2560,3.3486859639485678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,2560,6.231818644205729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,3072,4.893573506673177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,3584,5.583203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,2048,4.9850824991861975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,2048,2.8930145263671876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,1536,2.1555201212565107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,2560,4.365274556477865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,1024,1.603720474243164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,1024,2.679741923014323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,1536,3.8475936889648437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,768,1.2665578206380208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,768,2.0140959421793623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,512,1.4513460795084634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,2048,3.538255055745443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,512,1.0267285029093425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,256,0.9060266494750977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,1536,2.8805610656738283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,256,0.7205439885457356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,128,0.6968618392944336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,128,0.6638271967569987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,256,1.2249034881591796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,64,0.6486805597941081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,1024,2.1209760030110676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,64,0.6439200083414713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,32,0.6281909306844076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,51200,32,0.626691182454427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,768,1.7827477773030598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,512,1.4409610748291015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,51200,128,1.098086420694987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,51200,65536,154.752294921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,16384,7.379036966959636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,16384,11.952069091796876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,16384,7.760763549804688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,51200,22.430769856770834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,12288,9.028814697265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,12288,5.721156311035156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,65536,29.427239990234376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,10240,8.150820414225262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,10240,4.656437174479167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,12288,5.954566446940104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,51200,27.67957356770833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,8192,3.602715810139974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,10240,5.199541219075521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,8192,6.106001281738282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,51200,38.9501220703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,7168,3.2419509887695312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,8192,4.0804191589355465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,7168,5.516235860188802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,6144,2.8299029032389322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,65536,39.72100830078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,6144,4.481601969401042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,5120,2.4444905598958333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,5120,3.8509376525878904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,4096,1.877611796061198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,7168,3.647745005289713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,4096,3.1872459411621095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,6144,3.136017100016276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,3584,1.7291744232177735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,3072,1.4005610148111978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,3584,2.7276351928710936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,5120,2.5763926188151043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,3072,2.535308837890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,2560,1.213052749633789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,65536,49.95909016927083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,2560,1.9166431427001953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,4096,2.133465576171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,2048,1.650059763590495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,2048,1.1629493713378907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,1536,0.7951648076375325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,3584,1.9261962890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,1536,1.1833258310953776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,1024,0.557758903503418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,1024,0.9759637196858725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,768,0.6393717447916667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,768,0.4494933446248372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,3072,1.6110847473144532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,512,0.4656501452128093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,512,0.3231093406677246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,256,0.27836373647054036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,768,0.8329077402750651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,256,0.247926394144694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,2560,1.392357381184896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,128,0.22599786122639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,128,0.22659947077433268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,64,0.21163946787516275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,64,0.21783787409464517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,16384,32,0.2010218620300293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,16384,32,0.2119904041290283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,2048,1.1458901723225912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,1536,0.9274741490681967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,1024,0.6451658884684245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,512,0.4711872100830078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,256,0.3994367917378744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,16384,128,0.3700757344563802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,16384,5.367124430338541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,16384,8.791080729166667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,16384,5.850260416666666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,12288,7.8433878580729175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,51200,18.16050821940104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,12288,4.0448448181152346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,65536,22.39719034830729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,10240,5.636164347330729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,51200,20.140299479166664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,10240,3.9207445780436196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,12288,4.543281046549479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,8192,2.760766855875651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,8192,4.9233759562174475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,10240,3.8681877136230467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,51200,28.787990315755206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,7168,2.451885986328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,7168,3.901279958089193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,8192,3.1863563537597654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,6144,2.216936492919922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,65536,28.698824055989583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,6144,3.326562245686849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,5120,2.8209088643391924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,5120,1.7329962412516278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,4096,1.3995391845703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,4096,2.405927530924479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,7168,2.8360969543457033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,3584,1.2665227254231772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,3584,1.96703364054362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,6144,2.378460693359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,3072,1.0998719533284507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,3072,1.6970485687255858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,65536,37.59101969401042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,2560,1.4464149475097656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,5120,2.003968048095703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,2560,0.912115224202474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,2048,0.7908330917358398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,2048,1.162757364908854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,4096,1.5880831400553386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,1536,0.5802080154418945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,1536,1.0958602905273438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,1024,0.6193706512451171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,1536,0.744214375813802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,1024,0.41268266042073565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,768,0.4557216008504231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,768,0.3471637407938639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,1024,0.7656821568806966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,3584,1.3425866444905599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,512,0.25865279833475746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,512,0.33789653778076173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,256,0.1965845266977946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,3072,1.2276256561279297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,256,0.20128533045450844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,128,0.16664533615112304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,128,0.17669013341267903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,64,0.16145599683125814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,64,0.1714197317759196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,12288,32,0.15243733723958333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,2560,1.041258684794108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,12288,32,0.1661461353302002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,2048,0.8206453323364258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,768,0.4481247901916504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,512,0.3685375849405924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,256,0.3038250605265299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,12288,128,0.27833598454793296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,16384,4.634937540690104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,16384,7.353363037109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,12288,6.107624308268229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,16384,5.123691813151042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,51200,15.12982177734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,65536,19.45234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,51200,18.47421671549479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,12288,3.886522674560547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,10240,4.761271667480469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,10240,2.9580767313639322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,12288,3.8888928731282553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,8192,3.8380714416503907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,8192,2.3893109639485677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,51200,24.186505126953126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,7168,2.224416097005208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,7168,3.2280502319335938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,10240,3.3274709065755212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,65536,24.97117716471354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,6144,2.7776532491048176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,6144,1.8697333017985027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,8192,2.679221343994141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,5120,2.2734784444173175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,5120,1.4961322784423827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,4096,1.20184326171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,4096,1.9813855489095051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,7168,2.355246988932292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,65536,30.583762613932294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,3584,1.0769770304361979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,3584,1.6722751617431642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,6144,1.971962610880534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,3072,1.4766644795735677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,3072,0.8976608276367187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,2560,0.803608512878418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,2560,1.3349365234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,2048,0.9457802454630533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,5120,1.625482686360677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,2048,0.6445770899454752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,4096,1.247442118326823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,1536,0.570579210917155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,1536,0.7430954615275065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,1536,0.5998816172281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,1024,0.370743465423584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,1024,0.508737055460612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,768,0.36390508015950523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,768,0.2915445327758789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,512,0.28599573771158854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,3584,1.2040640513102212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,512,0.21357439359029134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,256,0.18227945963541667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,256,0.16589867273966472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,128,0.1448736031850179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,128,0.1503605365753174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,3072,0.9556959788004556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,64,0.13503360748291016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,2560,0.8327114741007486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,64,0.1454485257466634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,10240,32,0.12807573477427164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,10240,32,0.14215680758158367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,2048,0.6937525431315105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,1024,0.4210368156433105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,768,0.373303476969401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,512,0.3131648063659668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,256,0.2540810743967692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,10240,128,0.23827519416809081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,16384,3.9991424560546873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,16384,6.107561747233072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,12288,4.589692687988281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,16384,4.255689493815104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,51200,12.4207275390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,65536,16.846045939127603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,12288,3.661179860432943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,12288,3.220000966389974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,51200,13.164600626627603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,10240,4.151240539550781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,10240,2.64740473429362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,8192,3.0033643086751303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,51200,20.872347005208333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,8192,2.4448277791341146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,7168,2.731378173828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,7168,1.749852752685547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,10240,2.779622395833333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,6144,1.555566914876302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,6144,2.3622891743977865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,65536,21.165250651041667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,8192,2.1501609802246096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,65536,25.069930013020834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,5120,1.973531723022461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,5120,1.2850613911946616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,4096,1.5327093760172525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,4096,1.0551722844441733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,7168,1.921514638264974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,3584,1.4157535552978515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,3584,0.9563157399495443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,6144,1.5768661499023438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,3072,1.2192394256591796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,3072,0.8339167912801108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,2560,0.6451818466186523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,2560,1.0066368103027343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,2048,0.7240874608357747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,5120,1.3332330067952474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,2048,0.5361781438191732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,1536,0.6219722747802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,1536,0.4888725280761719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,4096,1.0740788777669272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,1024,0.4228831926981608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,1024,0.3195199966430664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,768,0.33054399490356445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,768,0.2514122645060221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,3584,0.9143125534057617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,512,0.22002453804016114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,512,0.20576319694519044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,256,0.13648319244384766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,256,0.14075627326965331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,3072,0.8330506642659505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,128,0.11536853313446045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,2560,0.6794549306233724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,128,0.1269034703572591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,64,0.10678293704986572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,64,0.12259946664174397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,8192,32,0.10212480227152507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,8192,32,0.11796800295511882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,1536,0.4572736104329427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,1024,0.3479733467102051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,768,0.3064319928487142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,2048,0.5662399927775066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,512,0.2539221286773682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,256,0.20849173863728843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,8192,128,0.19036693572998048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,16384,3.6319796244303384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,16384,5.194783020019531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,12288,3.9077588399251306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,51200,11.044986979166667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,16384,3.750872548421224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,12288,2.7322474161783856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,65536,15.003724161783854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,51200,11.547000122070312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,10240,3.2852447509765623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,10240,2.7159189860026043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,10240,2.5142751057942707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,51200,17.77450154622396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,12288,2.912344614664714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,8192,1.799725850423177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,8192,2.060416030883789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,8192,3.235380299886068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,7168,1.6005888621012372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,65536,16.714688110351563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,6144,1.3659722646077475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,6144,2.1361770629882812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,5120,1.1994528452555338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,7168,2.3580182393391924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,5120,1.7454549153645833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,65536,22.244183349609376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,4096,1.3522954305013022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,4096,0.9018357594807943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,4096,0.9545354843139648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,3584,0.823518943786621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,3584,1.1394634246826172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,3584,1.027132797241211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,3072,0.7516000111897786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,3072,1.058028793334961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,7168,1.6189226786295574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,2560,0.8490528106689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,2560,0.7036106745402019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,2048,0.5237087885538737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,2048,0.6700735727945963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,6144,1.4430314381917317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,1536,0.6148906707763672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,1536,0.38784958521525065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,1024,0.3814720153808594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,5120,1.171838887532552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,1024,0.3527167956034342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,768,0.2846720059712728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,768,0.2533909320831299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,512,0.2061024030049642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,512,0.18783893585205078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,256,0.1170954704284668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,256,0.12541013558705646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,3072,0.7100362777709961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,128,0.10287360350290935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,2560,0.6212736129760742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,128,0.11484373410542806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,64,0.09563199679056802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,2048,0.5110058784484863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,64,0.109279998143514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,1536,0.41231358846028643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,7168,32,0.09022506872812906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,7168,32,0.10606826941172283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,1024,0.3157877286275228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,768,0.2795647939046224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,512,0.22706133524576821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,256,0.18760852813720702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,7168,128,0.17194560368855794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,16384,3.200138600667318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,16384,4.5609888712565105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,12288,3.3478548685709635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,51200,9.92156473795573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,65536,12.696964518229167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,12288,2.422137705485026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,16384,3.375958506266276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,51200,9.668616739908854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,51200,14.431119791666665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,65536,15.670160929361979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,10240,2.0606549580891924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,10240,2.984056599934896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,12288,2.48087895711263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,8192,1.7716468811035155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,8192,2.213278961181641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,7168,1.4171562194824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,7168,2.3884511311848957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,65536,19.191756184895834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,6144,1.3998527526855469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,6144,1.7765771230061849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,10240,2.1135562896728515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,5120,1.5333653767903646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,5120,1.0051562627156576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,8192,1.6034805297851562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,4096,1.160855484008789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,4096,0.8258815765380859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,3584,1.0139018376668294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,3584,0.7585280100504558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,3072,0.8486719767252604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,3072,0.6723477045694988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,7168,1.4942090352376303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,2560,0.7444661458333333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,2560,0.5593866984049479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,6144,1.1429749806722005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,2560,0.637774912516276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,2048,0.5939456303914388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,2048,0.4317056020100911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,1536,0.4273205439249675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,2048,0.5156266530354817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,1536,0.34961067835489906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,5120,1.0111477533976236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,1024,0.3008042653401693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,1024,0.2536383946736654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,768,0.24652585983276368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,512,0.17954986890157063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,768,0.21421759923299155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,512,0.15788052876790365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,4096,0.8228053410847982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,256,0.10560533205668132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,256,0.1188927968343099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,128,0.08391893704732259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,3584,0.7140096028645833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,128,0.10345706939697266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,64,0.0779146671295166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,64,0.09952747027079265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,6144,32,0.08192000389099122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,3072,0.6053600311279297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,6144,32,0.09661226272583008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,1536,0.35504318873087565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,1024,0.27109333674112956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,768,0.24013226826985679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,512,0.19770666758219402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,256,0.15801280339558918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,6144,128,0.14598612785339354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,16384,2.80972900390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,16384,3.7197364807128905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,12288,2.7692278544108073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,51200,8.704613240559896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,51200,8.64595947265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,12288,2.1233076731363933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,65536,11.642793782552083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,16384,2.8958506266276043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,10240,1.7729173024495442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,10240,2.3804779052734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,51200,11.637357584635417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,8192,1.84464111328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,8192,1.4289663950602214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,7168,1.2941387176513672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,65536,16.077824910481773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,7168,1.9809354146321614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,65536,11.01404317220052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,12288,2.122414906819661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,6144,1.1438165028889975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,6144,1.4829855600992838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,10240,1.8422218322753907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,5120,0.9380853017171225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,5120,1.1989098866780599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,4096,1.025500806172689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,4096,0.7673760096232096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,3584,0.8840384165445963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,8192,1.4140939076741537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,3584,0.6888906478881835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,3072,0.7556938807169596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,7168,1.247225570678711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,3072,0.5657536188761394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,2560,0.4828330675760905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,2560,0.6419904073079427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,6144,1.0218698501586914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,2048,0.5093418757120769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,2048,0.41846081415812175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,5120,0.8249216079711914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,1536,0.36106561024983724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,4096,0.6812362670898438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,1536,0.32339839935302733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,1024,0.25065919558207195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,1024,0.23390506108601888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,768,0.19879573186238605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,1024,0.263753604888916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,3584,0.6249226888020833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,768,0.18949653307596842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,512,0.1425546646118164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,512,0.14079999923706055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,256,0.09327999750773111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,256,0.09975679715474448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,128,0.0738368034362793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,128,0.08653120199839273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,3072,0.5351797103881836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,64,0.06851306756337484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,64,0.0840885321299235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,5120,32,0.06826879978179931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,5120,32,0.08190293312072754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,2560,0.4520575841267903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,2048,0.382317860921224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,1536,0.3075519879659017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,768,0.20584319432576498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,512,0.1688661257425944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,256,0.13590186436971027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,5120,128,0.1259050687154134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,16384,2.456452178955078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,16384,3.19236577351888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,12288,2.2696489969889324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,51200,7.6393473307291675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,12288,1.8418516794840496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,65536,9.958740234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,16384,2.2825770060221355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,51200,9.371388753255207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,12288,1.7854816436767578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,10240,1.9455691019694012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,10240,1.542563247680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,10240,1.4593376159667968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,8192,1.4952074686686196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,65536,12.61656494140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,51200,7.200735982259114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,8192,1.240031941731771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,7168,1.486627197265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,7168,1.0999306996663412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,65536,9.78943583170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,6144,0.9689237594604492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,6144,1.3435050964355468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,5120,0.7881141026814779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,5120,1.073785654703776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,4096,0.7771370569864909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,4096,0.786907704671224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,3584,0.7229386647542317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,3584,0.6182602564493815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,3072,0.501254399617513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,3072,0.5999413172403971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,8192,1.163578669230143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,2560,0.5088298797607422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,2560,0.4437258720397949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,7168,1.0076074600219727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,6144,0.8307797114054362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,2048,0.3719882647196452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,2048,0.3740309397379557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,1536,0.29464426040649416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,1536,0.2799584070841471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,1024,0.19990933736165364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,5120,0.6985621134440104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,1024,0.19959146181742352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,4096,0.5680032094319661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,768,0.1494058609008789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,768,0.15235519409179688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,3584,0.5081664085388183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,512,0.11119039853413899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,512,0.11551893552144368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,256,0.0686463991800944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,256,0.08509546915690104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,3072,0.44978240331013997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,128,0.059825066725413004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,128,0.07529493172963461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,2560,0.3778250694274902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,64,0.05553919871648153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,64,0.07219733397165934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,4096,32,0.05328853527704874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,2048,0.3189706802368164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,4096,32,0.06952426433563233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,1536,0.25689813296000164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,1024,0.19428159395853678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,768,0.1696448008219401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,512,0.1389845371246338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,256,0.1118016004562378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,4096,128,0.10056320031483967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,16384,2.8073824564615886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,16384,2.2977867126464844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,51200,7.048298645019531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,51200,8.061604309082032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,65536,9.145998128255208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,12288,2.09124272664388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,12288,1.6734123229980469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,65536,10.3658935546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,12288,1.55206298828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,10240,1.722480010986328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,16384,2.0518783569335937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,10240,1.4592352549235026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,10240,1.2928885142008464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,8192,1.2886581420898438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,8192,1.133092244466146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,51200,6.616080220540364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,65536,8.723646036783855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,7168,1.0230282465616862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,7168,1.1959412892659507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,6144,0.8806933085123697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,6144,1.1691541035970052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,5120,0.8133984247843424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,5120,0.7409631729125976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,4096,0.7548095703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,4096,0.7037514368693034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,3584,0.5560906728108723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,3584,0.6389045079549154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,3072,0.5206720034281414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,3072,0.5410421371459961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,8192,0.9944543838500977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,2560,0.47505919138590497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,2560,0.3994282722473145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,7168,0.8762293497721354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,6144,0.7680927912394206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,2048,0.3261162757873535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,2048,0.32126827239990235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,1536,0.2544501304626465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,1536,0.24816533724466958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,5120,0.6289951960245769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,1024,0.1658026695251465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,1024,0.1771370728810628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,768,0.13039466540018718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,4096,0.5194687843322754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,768,0.14081279436747235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,768,0.159442138671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,3584,0.462008539835612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,512,0.09631786346435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,256,0.06131626764933268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,512,0.10653653144836425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,256,0.07777386506398519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,128,0.0538154681523641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,128,0.06647359927495321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,3072,0.4108810742696126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,64,0.04954453309377034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,64,0.064573868115743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3584,32,0.04748053153355916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3584,32,0.06386986573537191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,2560,0.3447392145792643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,2048,0.2916053454081217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,1536,0.2341258684794108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,1024,0.17985280354817706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,512,0.12803520361582438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,256,0.1018293301264445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3584,128,0.09166826407114664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,16384,2.2734921773274737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,16384,2.0419520060221354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,12288,1.6901098887125652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,12288,1.5736373901367187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,51200,6.40138651529948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,51200,7.098096211751302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,65536,8.246194966634114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,65536,9.351346842447917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,10240,1.4808074951171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,16384,1.875000508626302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,10240,1.317347208658854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,8192,1.1634207407633463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,8192,1.056169573465983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,7168,1.0061610539754233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,51200,5.746577962239583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,7168,0.9577258427937826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,6144,0.9080501556396484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,65536,7.7954259236653645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,6144,0.8150474548339843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,12288,1.337774912516276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,5120,0.784656016031901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,5120,0.5773461023966472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,5120,0.6905279795328776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,10240,1.0932640075683593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,4096,0.5875071843465169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,3584,0.4882464090983073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,4096,0.5976853052775065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,3584,0.4865877469380696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,3072,0.44745601018269854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,3072,0.43346347808837893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,8192,0.9005013147989909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,3072,0.3648607889811198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,2560,0.3801653226216634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,2560,0.35685866673787436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,2048,0.27554985682169597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,2048,0.3160191853841146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,1536,0.21754986445109048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,7168,0.771067746480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,1536,0.23621439933776855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,1024,0.14789546330769857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,1024,0.16557973225911457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,6144,0.6929066975911458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,768,0.12008000214894612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,768,0.1407807985941569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,4096,0.45441172917683914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,512,0.08917866547902426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,512,0.10274346669514973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,3584,0.40803734461466473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,256,0.05450453360875448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,256,0.07147093613942465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,2560,0.30511786142985026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,128,0.04745599826176961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,128,0.06086080074310303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,2048,0.2590997378031413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,64,0.04348160028457641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,64,0.05895893176396688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,3072,32,0.04190400044123332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,3072,32,0.05774720112482706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,1536,0.21004907290140787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,1024,0.15981440544128417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,768,0.13986239433288575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,512,0.11194453239440919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,256,0.08855040073394775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,3072,128,0.07999573548634847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,16384,1.891291681925456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,16384,1.8502260843912761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,12288,1.4299616495768228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,51200,5.735238647460937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,51200,5.18246815999349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,51200,5.9069056193033855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,65536,7.255609639485677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,12288,1.417406972249349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,65536,7.443317159016926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,10240,1.2049332936604817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,10240,1.1914496103922525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,8192,0.8830421447753907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,16384,1.573633066813151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,8192,0.9394869486490884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,7168,0.8262016296386718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,7168,0.8705600102742513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,6144,0.6737621307373047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,6144,0.7820522944132487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,5120,0.6189322789510091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,65536,6.644697570800782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,5120,0.6161759694417317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,4096,0.47133865356445315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,12288,1.192635726928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,4096,0.4952554702758789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,3584,0.4319658597310384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,10240,0.9599637349446615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,3584,0.4384949366251628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,3072,0.35631252924601237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,8192,0.7884970982869466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,3072,0.3804682731628418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,2560,0.28473173777262367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,2560,0.3234272003173828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,7168,0.6688789367675781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,2048,0.2313770612080892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,2048,0.279368527730306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,6144,0.5862485249837239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,1536,0.21754345893859864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,5120,0.4929717381795247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,1024,0.12354880174001057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,1024,0.15306240717569988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,4096,0.40690453847249347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,768,0.09997973442077637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,1536,0.17492693265279133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,768,0.12409599622090657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,512,0.07517866293589273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,3584,0.36093438466389977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,512,0.09227413336435954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,256,0.0502239982287089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,256,0.06525439818700154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,3072,0.3152618726094564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,128,0.039434667428334555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,128,0.05427519877751669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,2560,0.2670442581176758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,64,0.03587626616160075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,64,0.05259519815444946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2560,32,0.036057599385579425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,2048,0.23109973271687828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2560,32,0.05168639818827311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,1536,0.18340373039245605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,1024,0.1385493278503418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,768,0.11988693078358967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,512,0.09595733483632406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,256,0.07639040152231852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2560,128,0.06840960184733072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,16384,1.5087317148844401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,16384,1.6830496470133465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,12288,1.1024052937825521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,65536,5.953345235188802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,51200,4.576775614420573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,65536,6.686697387695313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,51200,5.200169372558594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,12288,1.3083104451497396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,10240,0.9184202829996744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,8192,0.7421887715657551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,10240,1.0559765497843423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,16384,1.3944095611572265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,8192,0.9082005182902018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,7168,0.6619605382283529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,65536,5.636068216959635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,7168,0.7539136250813802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,51200,4.459745279947916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,6144,0.5759040196736653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,12288,1.0347925186157227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,6144,0.645034662882487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,5120,0.4833770751953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,5120,0.5621418635050456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,4096,0.34453118642171227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,10240,0.8337013244628906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,4096,0.4714666684468587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,3584,0.3182154655456543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,3072,0.29154879252115884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,3584,0.4242570559183757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,3072,0.3698453267415365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,8192,0.653823979695638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,2560,0.24679999351501464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,2560,0.30520426432291664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,2048,0.1890229384104411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,2048,0.2519807974497477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,7168,0.5789450963338216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,1536,0.1394986629486084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,1536,0.19734400113423664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,6144,0.4980234781901042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,1024,0.09815680185953776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,1024,0.13427519798278809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,5120,0.42598826090494796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,768,0.07773439884185791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,768,0.10738666852315266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,4096,0.3505056063334147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,512,0.057974398136138916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,3584,0.30440212885538737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,512,0.08081173102060954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,256,0.03802880048751831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,256,0.057937065760294594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,3072,0.26684373219807944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,128,0.0325162669022878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,128,0.047856001059214275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,2560,0.22868266105651855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,64,0.030019199848175047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,64,0.046486401557922365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,2048,32,0.030082132418950396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,2048,32,0.0457856019337972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,2048,0.18952107429504395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,1536,0.15472426414489746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,1024,0.11690666675567626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,768,0.09971520105997721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,512,0.07967573006947835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,256,0.06234666506449381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,2048,128,0.055027198791503903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,16384,1.0984864552815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,16384,1.4969771067301432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,65536,4.34570566813151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,51200,3.601553090413412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,12288,0.7819007873535156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,12288,1.1311679840087892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,10240,0.6909439722696941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,65536,5.936549377441406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,51200,4.641350301106771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,10240,0.9415840148925781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,8192,0.5130709330240886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,7168,0.4636650721232097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,8192,0.7568031946818035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,16384,1.168278376261393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,7168,0.6947327931722005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,6144,0.4111477216084798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,65536,4.719437662760416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,6144,0.5803829193115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,51200,3.644388326009115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,12288,0.8358314514160157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,5120,0.3490111986796061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,4096,0.26382187207539876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,5120,0.48670291900634766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,3584,0.2278986612955729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,4096,0.41773331960042315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,10240,0.7040170669555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,3584,0.3507786750793457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,3072,0.2039466698964437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,3072,0.3049738566080729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,8192,0.5544490814208984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,2560,0.16577067375183105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,2560,0.26386772791544594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,2048,0.13585386276245118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,2048,0.21453653971354164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,7168,0.4829440116882324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,1536,0.1049610694249471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,1536,0.16928426424662274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,6144,0.42621227900187175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,1024,0.07585386435190836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,1024,0.12178346316019695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,5120,0.355570125579834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,768,0.05947519938151041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,4096,0.28959681193033854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,768,0.0957258701324463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,768,0.08258986473083496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,512,0.044684799512227376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,3584,0.26094613075256345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,512,0.07144746780395508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,256,0.030368000268936157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,3072,0.2265557289123535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,256,0.051526399453481045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,128,0.025860265890757246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,128,0.042395734786987306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,2560,0.1921610673268636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,64,0.02416213353474935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,64,0.040423464775085446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1536,32,0.02518933415412903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1536,32,0.03934933344523112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,2048,0.160808531443278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,1536,0.1299338658650716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,1024,0.09749440352121988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,512,0.06597973505655924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,256,0.0502239982287089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1536,128,0.044454399744669595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,65536,2.9855913798014324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,16384,0.6811050415039063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,51200,2.4343798319498697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,12288,0.5460853576660156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,16384,1.3292149861653646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,10240,0.42957334518432616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,12288,1.0512565612792968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,10240,0.8589290618896485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,65536,5.297027079264323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,51200,4.138136545817057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,8192,0.3460906664530436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,7168,0.30457706451416017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,8192,0.6742303848266602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,7168,0.6168160120646159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,16384,0.8620223999023438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,6144,0.2604416052500407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,12288,0.655738639831543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,65536,3.6563051859537756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,5120,0.22028800646464028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,51200,2.8725067138671876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,4096,0.18422187169392903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,5120,0.4427680015563965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,10240,0.5584661483764648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,3584,0.159170134862264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,4096,0.3657024065653483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,6144,0.5262432098388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,3584,0.3252511978149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,3072,0.13305919965108234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,3072,0.2757685343424479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,8192,0.4416821479797363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,2560,0.11640533606211345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,2560,0.2291818618774414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,2048,0.09346559842427572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,7168,0.3877450625101725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,2048,0.18898666699727376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,1536,0.07258346875508627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,1536,0.14870400428771974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,6144,0.3403978665669759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,1024,0.05199893315633138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,1024,0.10797119935353597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,5120,0.2854997316996256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,768,0.04197440147399902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,768,0.08563199838002523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,4096,0.23340907096862792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,512,0.03204693396886189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,3584,0.20945919354756676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,512,0.06298240025838217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,256,0.022536534070968627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,256,0.043619199593861895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,256,0.03834559917449951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,128,0.01704853375752767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,128,0.03412266572316487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,2560,0.1559050718943278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,64,0.015752533078193666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,64,0.03165439963340759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,2048,0.12812906901041668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,1024,32,0.015733333428700765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,1024,32,0.030876799424489336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,1536,0.1018229325612386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,3072,0.1810954729715983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,1024,0.07596266269683838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,768,0.06396160125732422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,65536,2.3962987263997393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,512,0.050052265326182045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,51200,1.7699850718180339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,1024,128,0.03242773413658142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,16384,0.5993557612101237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,12288,0.38508373896280923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,16384,1.2383946736653646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,12288,0.6141216278076171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,12288,0.948309326171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,10240,0.7928319931030273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,51200,3.872352091471354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,65536,4.947920227050782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,8192,0.25979839960734047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,7168,0.22822720209757485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,10240,0.3251999855041504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,8192,0.6322538375854492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,7168,0.5660959879557292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,6144,0.19604585965474447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,16384,0.7622186660766601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,6144,0.4806506792704265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,65536,3.135315195719401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,5120,0.16468267440795897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,51200,2.462577056884766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,5120,0.4127786636352539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,4096,0.13845760027567547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,10240,0.48580268224080403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,4096,0.33103361129760744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,3584,0.12117119630177815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,3584,0.29241600036621096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,3072,0.10302186806996663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,3072,0.2535328070322672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,8192,0.3897472063700358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,2560,0.08694187005360922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,2560,0.21437439918518067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,7168,0.34660161336263023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,2048,0.07194560368855794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,2048,0.1779861291249593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,3584,0.1857632001241048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,1536,0.057905066013336184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,1536,0.1396351973215739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,1024,0.040218667189280195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,6144,0.2996405283610026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,1024,0.10099306901295979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,768,0.03276053269704183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,768,0.07880427042643229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,5120,0.25114453633626305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,512,0.025552000602086383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,512,0.05845119953155518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,4096,0.20766612688700356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,256,0.01824000080426534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,256,0.0399125337600708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,3072,0.15940052668253582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,128,0.013794133067131042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,128,0.03111039996147156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,2560,0.13398079872131347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,64,0.011892267068227132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,64,0.027613866329193115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,768,32,0.012597333391507468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,768,32,0.027837866544723512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,2048,0.1123306671778361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,1536,0.0894442637761434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,1024,0.06539520025253295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,768,0.05417706569035848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,512,0.04275306860605876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,65536,1.549626668294271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,51200,1.147271474202474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,256,0.030827732880910237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,16384,0.3957311948140462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,12288,0.2684298515319824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,768,128,0.02582613428433736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,16384,1.1713429768880208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,10240,0.21593599319458007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,12288,0.8759040196736654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,51200,2.1000010172526045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,8192,0.17688852945963543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,10240,0.7347775777180989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,8192,0.5971829096476238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,51200,3.6065195719401038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,7168,0.15455466906229656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,65536,4.618537394205729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,6144,0.1359776020050049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,7168,0.5160170555114746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,6144,0.4442698796590169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,5120,0.11303679943084717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,16384,0.653933842976888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,5120,0.37236267725626626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,4096,0.09189653396606445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,65536,2.735020701090495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,4096,0.30167147318522136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,3584,0.0812544027964274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,3584,0.267521063486735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,8192,0.34192107518514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,3072,0.06923946539560953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,12288,0.4974837303161621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,3072,0.2322015921274821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,2560,0.05865919987360636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,3072,0.13608533541361492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,7168,0.30159041086832683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,2560,0.19808640480041503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,2048,0.04835413297017415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,10240,0.42173118591308595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,1536,0.03780373334884644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,2048,0.16299626032511394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,1536,0.12758293151855468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,1024,0.0276309331258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,1024,0.09074347019195557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,5120,0.21911360422770182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,768,0.022374399503072104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,768,0.07052799860636393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,4096,0.1821013291676839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,512,0.016753067572911583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,512,0.049532798926035564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,6144,0.2608703931172689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,256,0.011462400356928509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,256,0.02977173328399658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,3584,0.15899306933085125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,2560,0.11548373699188233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,128,0.009040000041325887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,128,0.02452053427696228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,2048,0.09632213115692138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,1536,0.07578559716542563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,64,0.007708799839019775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,512,32,0.00820906658967336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,64,0.022533333301544188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,512,32,0.021606399615605672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,1024,0.05471893151601156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,768,0.04571199814478556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,65536,0.8070218404134115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,51200,0.6272650400797526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,512,0.03494079907735188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,256,0.024126933018366496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,16384,0.20464000701904297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,12288,0.1606613318125407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,16384,1.0835712432861329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,12288,0.8159701029459635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,512,128,0.019605332612991334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,10240,0.1327903985977173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,12288,0.43372586568196614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,51200,1.7631093343098958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,8192,0.108187731107076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,10240,0.685148811340332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,7168,0.09462933540344239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,8192,0.5542432149251302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,51200,3.3720342000325525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,7168,0.48544960021972655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,65536,4.3044275919596355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,6144,0.08298880259195963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,5120,0.0699285348256429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,6144,0.22784105936686197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,6144,0.4189194679260254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,4096,0.057513598601023355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,5120,0.35357866287231443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,4096,0.28465919494628905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,3584,0.051463464895884194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,16384,0.5768960316975911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,65536,2.2266571044921877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,3584,0.25223679542541505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,3072,0.051077334086100254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,3072,0.2183039983113607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,2560,0.03929493427276611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,2560,0.18435413042704266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,10240,0.3696832021077474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,2048,0.03329919974009196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,8192,0.2978048006693522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,2048,0.1514623959859212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,1536,0.02649066646893819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,1536,0.1183509349822998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,7168,0.2649439970652262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,1024,0.01683093309402466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,1024,0.0825663963953654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,5120,0.19124587376912433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,768,0.012169599533081055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,4096,0.1544266700744629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,768,0.06129813194274902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,512,0.009809066851933796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,512,0.04016533295313517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,3584,0.13717652956644694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,256,0.007496533294518788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,256,0.02580159902572632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,3072,0.11852906545003254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,128,0.006305066744486491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,128,0.021002666155497233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,2560,0.09965973695119222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,64,0.0055754666527112326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,64,0.01973759929339091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,256,32,0.0057888001203536986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,256,32,0.019151999553044637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,2048,0.08106346925099692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,1536,0.06300373474756876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,65536,0.7856565475463867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,1024,0.04461546738942464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,51200,0.5853450775146485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,768,0.036373333136240645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,512,0.026474666595458985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,16384,0.19093119303385417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,256,0.01759679913520813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,12288,0.14645867347717284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,256,128,0.013902933398882548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,16384,1.073846435546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,51200,1.6923124949137371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,10240,0.12460373242696125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,12288,0.8070613225301108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,8192,0.0988373359044393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,10240,0.6807861328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,8192,0.5417621612548829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,7168,0.086954665184021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,51200,3.339335378011068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,65536,4.277537027994791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,7168,0.47608213424682616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,6144,0.08815680344899496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,6144,0.4108554522196452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,5120,0.06392639875411987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,16384,0.5475242614746094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,5120,0.34519147872924805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,4096,0.05275199810663859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,65536,2.1334805806477863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,4096,0.2789717356363932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,12288,0.41873919169108076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,3584,0.04684160153071086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,3584,0.24863999684651694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,10240,0.349019718170166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,3072,0.04129173358281453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,3072,0.2152181307474772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,8192,0.28416531880696616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,2560,0.03519573211669922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,7168,0.2475050608317057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,2560,0.18232107162475586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,2048,0.02950399915377299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,2048,0.14843200047810873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,6144,0.21425813039143882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,1536,0.023437867561976113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,1536,0.11379733085632324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,5120,0.17930132548014324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,1024,0.010743467013041179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,1024,0.07780906359354654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,4096,0.14650774002075195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,768,0.008756267031033833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,768,0.05636693239212036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,512,0.006905599931875865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,512,0.0360479990641276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,3584,0.1282080014546712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,256,0.005522133409976959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,256,0.02412266731262207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,3072,0.11039040088653565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,128,0.0046015997727712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,128,0.01958613395690918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,2560,0.09248639742533366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,64,0.004285866518815359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,64,0.018145066499710084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,128,32,0.0046965335806210835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,128,32,0.01752426624298096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,2048,0.07491520245869955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,1536,0.05651946862538656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,65536,0.7559296290079753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,1024,0.03903040091196696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,51200,0.5923594792683919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,16384,0.1865887959798177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,768,0.030988800525665283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,12288,0.14188586870829265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,512,0.021857066949208578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,10240,0.11910080115000407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,8192,0.09632213115692138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,256,0.014099199573198953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,7168,0.08508266607920328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,16384,1.0678187052408854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,12288,0.8080671946207681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,10240,0.6727743784586588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,8192,0.5404736200968425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,6144,0.07451840241750082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,7168,0.47704213460286454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,5120,0.06199359893798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16384,128,128,0.010221866766611735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,4096,0.050741334756215416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,3584,0.04490666786829631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,3072,0.03922986586888631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,2560,0.03339840173721313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,5120,0.3431349436442057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,4096,0.2787562688191732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,6144,0.4119690577189128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,3072,0.21229119300842286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,3584,0.24581546783447267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,2048,0.027297067642211913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,1536,0.021744000911712646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,1024,0.008983467022577922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,768,0.0071263998746871945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,1024,0.0751530647277832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,512,0.005718400080998739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,1536,0.112391463915507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,2048,0.1459221363067627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,256,0.004423466821511587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,768,0.054093865553538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,128,0.0038592000802357995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,512,0.03255680004755656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,64,0.003606399893760681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,256,0.023545600970586143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,64,32,0.0036821333070596062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,64,0.01718613306681315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,32,0.016660267114639284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,51200,0.5857280095418294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,65536,0.7440917332967122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,16384,0.18517440160115559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,12288,0.1413706620534261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,65536,4.2691650390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,51200,3.3311541239420572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,2560,0.17923946380615235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,8192,0.0958079973856608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,10240,0.12956906954447428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,64,128,0.019127466281255088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,7168,0.0841269334157308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,6144,0.07358933289845784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,7168,0.4765514691670735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,10240,0.6735647837320964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,12288,0.8033013025919595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,5120,0.061425065994262694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,6144,0.41072746912638347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,4096,0.05054826736450195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,3584,0.04446933269500732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,5120,0.3446133295694987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,4096,0.2782186508178711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,3072,0.038363734881083175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,3584,0.245522133509318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,2560,0.032680533329645794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,2048,0.026868265867233277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,3072,0.2119808038075765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,1536,0.021150932709376017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,2560,0.17866239547729493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,1024,0.007594666878382365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,2048,0.14582719802856445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,768,0.006557866434256236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,1536,0.11167893409729004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,1024,0.07488959630330404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,512,0.005211733281612396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,256,0.004183466732501984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,768,0.05268373489379883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,512,0.03302613298098246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,128,0.0035946667194366455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,256,0.02295680046081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,128,0.018578133980433145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,32,0.0033600000043710076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,32,0.016037333011627197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,51200,3.3290326436360678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,8192,0.5408704121907552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,65536,4.260599517822266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16384,32,64,0.0033013333876927694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,64,0.0167797327041626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16384,32,16384,1.0586175918579102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,12288,9.539687093098959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,16384,13.105049641927085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,16384,16.885784912109376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,12288,19.95754597981771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,16384,24.287060546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,10240,8.010461934407552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,12288,11.949790445963542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,10240,15.567368570963541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,10240,12.004117838541667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,8192,7.441459147135417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,8192,11.96552022298177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,7168,5.968793741861979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,8192,7.661712137858073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,7168,10.475950113932292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,51200,41.33031005859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,6144,5.122496032714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,7168,6.8058934529622395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,5120,4.370175170898437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,6144,9.862765502929687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,6144,5.710965474446615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,5120,8.147966003417968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,4096,6.253887939453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,4096,3.402118428548177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,4096,4.0167083740234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,5120,5.018992106119791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,3584,2.9425013224283854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,3584,5.29660898844401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,3072,2.512646484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,3584,3.8954121907552084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,3072,4.6581578572591145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,2560,2.5970303853352865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,2560,3.871454874674479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,2048,1.8433162689208984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,2560,2.7048235575358075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,2048,3.119574483235677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,1536,1.448529052734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,3072,3.309108225504557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,1536,2.529296112060547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,1024,1.6423221588134767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,1024,1.008673095703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,768,0.8113983790079752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,768,1.3665461222330728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,768,1.1572362263997396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,512,0.906109873453776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,512,0.6799466451009114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,2048,2.361480458577474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,256,0.4612064043680827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,256,0.5725247701009114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,1536,1.792782974243164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,128,0.44357439676920574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,128,0.42554772694905596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,64,0.4114272117614746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,51200,62.020975748697914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,32,0.40073493321736653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,1024,1.3643199920654296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,32,0.4527125358581543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,64,0.41286506652832033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,512,0.8972266515096029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,256,0.7677024205525715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,128,0.7228319803873698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,51200,77.50040690104166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,16384,21.160904947916666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,16384,10.493307495117188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,51200,33.71985677083333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,16384,11.61597900390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,12288,14.922364298502604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,12288,7.807902018229167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,65536,46.18578287760417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,12288,9.135709635416667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,51200,44.80708821614583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,10240,6.749627685546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,10240,12.408644612630209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,10240,8.511546834309895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,8192,5.264291381835937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,7168,5.127361043294271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,8192,10.27581075032552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,8192,6.0271453857421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,7168,8.411230977376302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,6144,3.8669578552246096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,7168,5.382185363769532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,6144,7.470689900716145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,5120,3.292979176839193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,51200,68.8543212890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,5120,7.032887268066406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,5120,3.8419637044270836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,6144,4.491234334309896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,65536,66.59488118489584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,4096,2.961279042561849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,3584,2.335399373372396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,4096,5.223446146647135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,3072,2.129083760579427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,3584,4.271160380045573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,3072,3.7462303161621096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,3072,2.8214261372884115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,4096,3.224850209554037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,2560,1.8312299092610675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,2560,2.0903594970703123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,3584,2.841625722249349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,2560,3.342244211832682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,2048,1.4735392252604167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,2048,2.607476298014323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,1536,1.065780258178711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,1536,1.8471115112304688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,1024,0.7916309356689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,1024,1.3241002400716146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,768,0.6575114568074544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,768,1.019271469116211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,512,0.7318527857462566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,512,0.49260800679524736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,256,0.4496575991312663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,256,0.36821438471476237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,128,0.35115092595418296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,128,0.34006080627441404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,2048,1.7931199391682942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,1536,1.3718772888183595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,128,0.5695903778076172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,64,0.32685972849527994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,32,0.3172671953837077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,64,0.3325930595397949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,1024,1.0206783930460612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,51200,32,0.32434345881144205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,768,0.8886645634969076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,512,0.7143871943155925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,51200,256,0.5954602559407551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,51200,65536,87.04925130208333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,16384,6.132628377278646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,16384,3.680141957600912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,51200,11.100670369466146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,65536,15.271150716145835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,12288,4.594996134440104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,12288,3.0567242940266928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,16384,4.17857920328776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,51200,12.590122477213542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,10240,2.325507100423177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,65536,20.17008056640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,10240,4.032487487792968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,8192,1.8871040344238281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,51200,20.880918375651042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,8192,3.4317248026529947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,10240,2.693074035644531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,12288,3.0983189900716144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,7168,1.6547669728597003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,7168,1.7321547190348308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,7168,3.106892903645833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,6144,1.4574347178141276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,6144,2.40950927734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,5120,1.1726698557535806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,5120,2.0054964701334637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,4096,0.9401461283365885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,8192,2.059304555257161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,4096,1.6194154103597005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,3584,0.8638069152832031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,65536,24.912040201822915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,3072,0.7254314422607422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,3072,1.1817131042480469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,6144,1.4637418111165366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,3072,0.7741514841715496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,3584,1.363382339477539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,2560,0.6103466669718425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,2560,0.9884895960489908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,2560,0.7538485209147135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,2048,0.4689311981201172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,2048,0.7613941192626953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,1536,0.4173706690470378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,1536,0.6304085413614909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,1024,0.439142386118571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,4096,1.0030752182006837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,1024,0.2979328155517578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,768,0.34885120391845703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,768,0.23638933499654136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,512,0.2524650732676188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,3584,0.9045440038045248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,512,0.19262293179829915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,5120,1.1850762685139975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,256,0.1363093376159668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,256,0.13468799591064454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,128,0.11540053685506184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,128,0.12359253565470378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,2048,0.5516970952351887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,64,0.10754880110422771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,1536,0.44547414779663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,64,0.12353280385335286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,32,0.10376319885253907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,1024,0.33175894419352214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,32,0.12031466960906982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,768,0.29944321314493816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,512,0.24550612767537436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,256,0.20297279357910156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,128,0.19005972544352215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,16384,2.77586669921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,16384,4.346121724446615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,12288,3.3072586059570312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,51200,8.381259663899739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,51200,9.179651896158855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,16384,3.150426737467448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,12288,2.0237237294514974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,65536,10.68365987141927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,10240,1.7239147186279298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,10240,2.959025065104167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,8192,2.7246241251627605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,8192,1.3728511810302735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,51200,14.483392333984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,12288,2.395893351236979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,7168,1.215721638997396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,8192,1.8100692749023437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,7168,2.1270005544026693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,10240,1.928473663330078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,6144,1.031607437133789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,6144,1.7784736633300782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,5120,0.9216992060343424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,5120,0.9487221399943033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,5120,1.4601941426595053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,65536,14.749217732747397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,4096,1.1017557779947915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,4096,0.7319733301798503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,65536,19.389689127604164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,3584,0.9975306828816732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,3584,0.7933386484781901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,3072,0.8721322377522787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,7168,1.254465103149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,3072,0.5325749397277832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,2560,0.5169088045756023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,2560,0.7711488087972005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,6144,1.1738763173421225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,2048,0.5288000106811523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,2048,0.3988767941792806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,1536,0.294539737701416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,1536,0.478601614634196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,1024,0.29184532165527344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,1024,0.24680426915486656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,768,0.2397696018218994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,768,0.18902079264322919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,4096,0.7609749476114909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,512,0.17733972867329914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,3584,0.6551093419392904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,512,0.15115307172139486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,256,0.10511039892832438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,256,0.10796480178833008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,3072,0.5699509302775065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,128,0.08706346352895102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,128,0.09869333108266196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,2560,0.48935680389404296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,128,0.15234880447387694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,64,0.07979520161946616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,32,0.07900479634602865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,64,0.09667200247446696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,2048,0.42082665761311844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,32,0.09431146780649821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,1536,0.33533013661702477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,1024,0.2572906653086344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,768,0.22863893508911132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,512,0.18949546813964843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,256,0.15578559239705403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,16384,2.397413380940755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,16384,3.889421844482422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,16384,2.6497889200846356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,51200,7.289685567220052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,12288,2.799515787760417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,12288,1.7879498799641926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,65536,9.76942850748698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,10240,1.5013482411702475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,10240,2.3644704182942706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,51200,11.418023681640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,8192,1.2048906962076822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,51200,7.746472676595052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,8192,1.9228853861490887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,12288,1.92628911336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,7168,1.0431615829467773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,7168,1.6599615732828776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,7168,1.096779759724935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,65536,11.190463256835937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,10240,1.6450069427490235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,6144,0.9367733637491862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,6144,1.4066293080647787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,65536,16.323936971028647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,5120,1.2079381306966146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,5120,0.8321610768636069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,4096,0.9428288141886393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,8192,1.210043716430664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,4096,0.6132949193318684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,3584,0.8835423787434896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,3584,0.5970613479614257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,3072,0.7234986623128254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,3072,0.5253546714782715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,2560,0.42826026280721025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,2048,0.45461333592732744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,6144,0.9562709172566732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,2048,0.33141441345214845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,1536,0.34775253931681316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,5120,0.7812672297159831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,2560,0.6408383687337239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,1536,0.27919146219889324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,1024,0.23843626976013182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,1024,0.20148800214131674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,4096,0.6196234385172527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,768,0.19733014106750488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,768,0.1561994711558024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,3584,0.547650146484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,512,0.140338134765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,768,0.2051914691925049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,512,0.13058026631673175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,256,0.09610666433970133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,512,0.16856640179951984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,256,0.09494079748789469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,128,0.0757536013921102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,128,0.08524479866027831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,64,0.06947306791941324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,64,0.08304959932963053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,32,0.06899840037027995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,32,0.08197759787241618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,2560,0.42111358642578123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,2048,0.34850025177001953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,3072,0.4810069402058919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,1536,0.2818261464436849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,1024,0.21818666458129882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,256,0.13084800243377687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,128,0.11947519779205322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,16384,1.9757952372233074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,16384,3.0976468404134114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,51200,6.332344563802083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,12288,2.2435585021972657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,65536,8.202669779459637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,12288,1.491389846801758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,16384,2.104625066121419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,51200,9.37459004720052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,10240,2.041611735026042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,10240,1.2829503377278646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,51200,6.3531041463216145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,8192,1.0162368138631186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,65536,12.064761352539062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,8192,1.5000757853190103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,7168,0.9662442525227866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,12288,1.5372650146484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,7168,1.3267648061116537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,7168,1.0947093963623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,65536,9.132625325520833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,6144,1.2156074523925782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,6144,0.8615829467773437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,5120,0.9738335927327475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,5120,0.7799381256103516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,4096,0.9242090861002603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,4096,0.6003445307413737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,10240,1.3654869079589844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,3584,0.7806058883666992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,3584,0.5212160110473633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,8192,1.0401568094889322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,3072,0.6299381256103516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,3072,0.41136213938395183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,3072,0.5072106679280599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,2560,0.3420703887939453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,2560,0.5192426681518555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,2048,0.4484885215759277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,2048,0.2666624069213867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,1536,0.23215146064758302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,1536,0.24265920321146645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,6144,0.7448533376057942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,1024,0.1952405293782552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,1024,0.17964372634887696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,5120,0.635978635152181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,768,0.1596511999766032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,768,0.13764479955037434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,512,0.11523306369781494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,4096,0.5077856063842774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,512,0.10305813153584797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,1536,0.3610015869140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,256,0.07010347048441569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,256,0.07887252966562906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,256,0.11032746632893879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,128,0.059646932284037266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,3584,0.46758079528808594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,128,0.07064212958017985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,128,0.09820480346679687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,64,0.05496319929758707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,32,0.05421546697616577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,64,0.07042346795399984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,32,0.06984000205993653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,2560,0.33727572758992513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,2048,0.29092585245768227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,1024,0.18054399490356446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,768,0.15948266983032228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,512,0.13250880241394042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,16384,2.732201639811198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,16384,1.8327220916748046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,65536,7.15142822265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,51200,5.495735677083333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,12288,1.3451786041259766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,51200,8.155442301432291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,12288,2.009362157185872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,65536,10.403018188476562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,10240,1.1172448476155599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,16384,1.8649631500244142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,10240,1.727350362141927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,65536,7.199068705240886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,51200,5.676492818196615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,8192,0.9246250788370768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,8192,1.3475626627604167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,7168,1.1635306040445963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,7168,0.8059850692749023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,12288,1.3577407836914062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,6144,1.0453450520833334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,6144,0.8307082494099935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,5120,0.8435573577880859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,5120,0.6275978724161784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,5120,0.6752511978149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,4096,0.48974294662475587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,4096,0.7735189437866211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,10240,1.228398895263672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,3584,0.42006079355875653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,3072,0.35512641270955403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,3072,0.5579573313395183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,8192,0.9229866663614908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,2560,0.4592405319213867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,2560,0.3405130704243978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,2560,0.3636042594909668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,7168,0.7721408208211262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,2048,0.3468298594156901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,2048,0.2782335917154948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,1536,0.25256640116373696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,6144,0.6835701624552409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,1536,0.21358079910278321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,1536,0.24455787340799967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,1024,0.1900426705678304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,1024,0.14921174049377442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,768,0.15049707094828288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,768,0.12091519832611083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,512,0.099180801709493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,512,0.0929365317026774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,4096,0.4593525250752767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,256,0.06595306793848674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,3584,0.7195754369099935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,256,0.07525226275126139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,128,0.05498239994049072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,128,0.06679466565450033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,3584,0.41362667083740234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,64,0.049745066960652666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,64,0.06543999910354614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,32,0.04842346509297689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,32,0.06378986835479736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,768,0.1805909315745036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,3072,0.3593536059061686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,2048,0.26286293665568033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,1024,0.16444692611694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,512,0.11946773529052734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,256,0.09813546339670817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,128,0.09052586555480957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,16384,1.5899637858072917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,16384,2.380348714192708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,12288,1.7100949605305988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,65536,6.419264221191407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,51200,4.969535827636719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,65536,6.745301310221355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,12288,1.2481204986572265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,10240,1.0794965108235677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,10240,1.4100821177164713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,51200,7.005098470052083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,65536,9.049473063151042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,8192,0.8358314514160157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,8192,1.118023427327474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,16384,1.58743044535319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,7168,0.7351680119832357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,7168,0.9829610824584961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,6144,0.9115680058797201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,51200,5.0887603759765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,6144,0.6203829447428386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,5120,0.74661865234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,12288,1.178217569986979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,5120,0.5427573521931965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,4096,0.4614549318949382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,4096,0.6002527872721355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,3584,0.41794452667236326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,10240,1.005571174621582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,3584,0.5065845489501953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,8192,0.7808725357055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,3072,0.4540245374043782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,3072,0.34939521153767905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,3072,0.36078081130981443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,2560,0.3624629338582357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,7168,0.6710474650065105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,4096,0.42757972081502277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,2560,0.2894528071085612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,2048,0.23087573051452637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,2048,0.3023029327392578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,1536,0.2147050698598226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,1536,0.19939947128295898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,1536,0.18694720268249512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,6144,0.579414431254069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,1024,0.1308351993560791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,5120,0.48810666402180986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,1024,0.14163200060526532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,768,0.11578133106231689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,512,0.08558080196380616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,768,0.11423786481221516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,512,0.0831765333811442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,256,0.05390826861063639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,256,0.06539733409881592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,1024,0.16575786272684734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,128,0.0478005329767863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,128,0.058584535121917726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,3584,0.35585918426513674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,64,0.043236267566680905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,64,0.05819626649220785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,32,0.042444801330566405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,32,0.05730666716893514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,2560,0.2695232073465983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,2048,0.22697812716166177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,512,0.1032981316248576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,768,0.1241429328918457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,256,0.08401493231455484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,128,0.07720533212025961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,16384,1.975185012817383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,16384,1.3866976420084636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,12288,1.4161301930745442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,65536,5.644900512695313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,51200,4.329201253255208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,51200,5.760372416178385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,12288,1.0995477040608725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,10240,1.1862069447835286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,65536,7.421631876627605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,10240,0.8951285044352213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,8192,0.918121592203776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,8192,0.7330986658732097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,16384,1.461303456624349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,7168,0.8134911855061849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,7168,0.6284565607706706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,51200,4.392610168457031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,65536,5.514267985026041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,6144,0.5466890970865885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,6144,0.698137601216634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,6144,0.5325034777323405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,5120,0.5881823857625326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,12288,1.0278538386027019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,5120,0.4568170547485352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,4096,0.38959039052327477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,4096,0.46692053476969403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,3584,0.389845339457194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,10240,0.8511615753173828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,3584,0.33877760569254556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,3072,0.36212266286214195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,3072,0.27685012817382815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,2560,0.3071722666422526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,3072,0.3184768040974935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,2048,0.24633280436197916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,2560,0.26591466267903646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,8192,0.663805898030599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,2048,0.20974186261494956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,1536,0.19212692578633625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,1536,0.16413547197977701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,1536,0.1821087996164958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,1024,0.12287786801656086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,1024,0.1252010663350423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,7168,0.5892223993937175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,3584,0.3627818743387858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,768,0.10154773394266765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,512,0.07558933099110922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,768,0.10905706882476807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,512,0.07839679718017578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,512,0.10564266840616862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,256,0.049985067049662275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,256,0.06286079883575439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,128,0.040742401281992593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,128,0.053179732958475744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,2560,0.246833070119222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,64,0.03614506721496582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,64,0.0514794667561849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,5120,0.41663252512613935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,32,0.035913598537445066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,32,0.051164801915486655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,4096,0.3470154762268066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,2048,0.19698239962259928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,1024,0.12630613644917804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,768,0.10956160227457683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,256,0.07268266677856446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,128,0.06694506804148356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,16384,1.5519893646240235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,65536,4.794605000813802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,65536,5.847283426920573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,16384,1.2650506337483725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,51200,4.636970520019531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,51200,3.7460245768229163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,12288,0.9458560307820638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,12288,1.0606175740559896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,10240,0.8543189366658529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,10240,0.7473141352335613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,8192,0.7422112147013347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,8192,0.6497685114542644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,65536,4.858360290527344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,7168,0.642084248860677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,7168,0.5743690490722656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,6144,0.5518175760904949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,6144,0.4773898760477702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,16384,1.1331104278564452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,5120,0.43079465230305986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,51200,3.695721689860026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,5120,0.44191573460896805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,4096,0.35558719635009767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,4096,0.32658987045288085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,12288,0.8027648289998373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,4096,0.31807359059651696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,3584,0.3040191968282064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,3584,0.2839167912801107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,10240,0.7137781143188476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,3072,0.28089920679728186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,3072,0.25048534075419104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,3072,0.27398080825805665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,2560,0.23547093073527017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,2560,0.21363306045532227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,2048,0.20201172828674316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,8192,0.5395061492919921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,2048,0.17599786122639974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,1536,0.13654507001241048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,2048,0.19109226862589518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,1536,0.13691627184549968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,1024,0.1092629353205363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,1024,0.10044480164845784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,768,0.07750933170318604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,7168,0.47989441553751633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,768,0.08362773259480795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,512,0.057606399059295654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,512,0.06618773142496745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,6144,0.41541439692179366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,256,0.039862398306528726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,256,0.05288213491439819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,256,0.06117120186487833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,128,0.03326080044110616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,128,0.04572800000508626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,5120,0.3546751976013184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,64,0.02909119923909505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,64,0.044495999813079834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,32,0.029846400022506714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,32,0.0448309342066447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,3584,0.25546347300211586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,2560,0.19517119725545246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,1536,0.1317855993906657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,1024,0.101800537109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,768,0.08860267003377278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,512,0.07317012945810954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,128,0.053808001677195225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,16384,1.2245749155680339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,16384,1.1180608113606771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,16384,1.0741877237955728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,65536,4.4761505126953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,51200,3.454151407877604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,12288,0.9622090657552084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,51200,3.3165918986002603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,65536,5.118480936686198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,51200,3.996617635091146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,65536,4.407925415039062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,12288,0.8665866851806641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,10240,0.7309631983439128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,10240,0.8280682881673178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,8192,0.6310293197631835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,8192,0.5948191960652669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,7168,0.5475701649983724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,7168,0.5159893353780111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,6144,0.4891754786173503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,6144,0.4354634602864583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,5120,0.38466558456420896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,5120,0.3188245455423991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,5120,0.39249601364135744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,4096,0.3166090647379557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,4096,0.30197652180989587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,3584,0.2791562716166178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,3584,0.26558186213175455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,3072,0.23074132601420083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,3072,0.2359861373901367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,12288,0.7340789159138997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,8192,0.5056181271870931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,2560,0.19760533968607585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,2560,0.20778986612955727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,7168,0.4352405230204265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,2048,0.1620906670888265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,6144,0.3784735997517904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,2048,0.17591892878214518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,1536,0.12855786482493084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,1536,0.13313173453013102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,4096,0.2636693318684896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,10240,0.6149930953979492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,1024,0.08571200370788574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,1024,0.09485119978586833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,1024,0.09532586733500162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,768,0.06896959940592448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,3584,0.23542292912801108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,768,0.0784447987874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,512,0.05110186735788981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,512,0.06222293376922607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,256,0.03389333486557007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,256,0.04771626790364583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,3072,0.20537919998168946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,128,0.029601067304611206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,128,0.04310400088628133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,128,0.04895253181457519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,64,0.02738453348477681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,64,0.041155199209849044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,32,0.02671146591504415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,32,0.042467200756073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,2560,0.17665173212687174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,2048,0.14778560002644855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,1536,0.12300159931182861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,768,0.08160746892293294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,512,0.06599466800689698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,256,0.054162132740020755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,16384,1.0568362553914388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,16384,1.0158271789550781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,65536,4.375096638997396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,12288,0.7654250462849934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,65536,4.091811116536459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,51200,3.142782847086589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,51200,3.5863850911458335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,12288,0.8092277526855469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,10240,0.6893973032633464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,10240,0.6486015955607096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,8192,0.5306410789489746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,8192,0.5644746780395508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,7168,0.4921248118082683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,7168,0.4743040084838867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,6144,0.3940544128417969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,16384,0.8833045323689779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,6144,0.43052053451538086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,5120,0.3306346575419108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,65536,3.810429890950521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,51200,2.9264244079589843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,5120,0.3659306526184082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,4096,0.30251731872558596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,12288,0.6485813140869141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,4096,0.28256425857543943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,10240,0.5495669047037761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,3584,0.2502240022023519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,3584,0.2493055979410807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,3072,0.2170581340789795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,3584,0.22939839363098144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,8192,0.4361269315083821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,3072,0.2156031926472982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,2560,0.17391786575317383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,2560,0.19017386436462402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,2048,0.14325226147969564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,2048,0.15328213373819988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,1536,0.10620799859364827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,7168,0.38037331899007165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,1536,0.11850773493448893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,1024,0.07530559698740641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,1024,0.0876682678858439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,1024,0.08459200064341227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,768,0.059987199306488034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,768,0.07299520174662272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,6144,0.336517333984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,512,0.0456928014755249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,512,0.059388800462087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,5120,0.2797024091084798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,256,0.030577067534128827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,256,0.04396266539891561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,4096,0.23127466837565103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,128,0.026231465737024943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,128,0.03966720104217529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,2560,0.15625279744466145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,3072,0.18218986193339032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,64,0.023594667514165245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,32,0.024706133206685386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,2048,0.13392000198364257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,64,0.039160533746083574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,32,0.038427734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,1536,0.10758933226267497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,768,0.07228906949361166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,512,0.05867093404134115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,256,0.04721493323644002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,128,0.043010131518046064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,16384,0.8409290949503581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,51200,2.861926523844401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,65536,3.3575477600097656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,16384,0.9301653544108073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,65536,3.668751017252604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,65536,3.842630513509115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,12288,0.6913600285847982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,51200,2.8605738321940106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,12288,0.7273312250773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,12288,0.5763455708821614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,10240,0.5852298736572266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,10240,0.6115605036417644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,8192,0.433513609568278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,8192,0.4945247968037923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,7168,0.37286507288614906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,7168,0.4397717475891113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,6144,0.32972691853841146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,6144,0.370796807607015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,5120,0.27426239649454753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,5120,0.3136373202006022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,16384,0.7523605346679687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,4096,0.21935253143310546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,51200,2.4747029622395833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,10240,0.4770496050516765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,4096,0.2699413299560547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,3584,0.21355733871459961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,3584,0.2275328000386556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,3072,0.17886826197306316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,8192,0.39399360020955404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,3072,0.17595094045003254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,3072,0.19837652842203776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,2560,0.14722453753153483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,2048,0.12092479864756267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,2560,0.13796159426371257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,2560,0.17450772921244304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,7168,0.33856639862060545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,2048,0.1481269359588623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,1536,0.09179519812266032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,1024,0.06628266572952271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,1536,0.10838293234507243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,1024,0.08240853150685629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,6144,0.292031987508138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,768,0.052819200356801356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,768,0.06700800259908041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,512,0.03944960037867228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,512,0.0538261334101359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,5120,0.247652260462443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,256,0.027777065833409626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,256,0.03971840143203735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,256,0.04057173331578572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,128,0.02177600065867106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,128,0.03514346679051717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,128,0.036518398920694986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,64,0.01952000061670939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,64,0.034221867720286056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,32,0.020338133970896403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,32,0.03375146786371867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,3584,0.18306667009989422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,4096,0.2039893309275309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,2048,0.11582079728444417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,1536,0.09517866770426432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,1024,0.07296000321706136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,768,0.0634335994720459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,512,0.05060799916585287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,16384,0.7024853388468425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,16384,0.8377877553304037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,65536,3.136557769775391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,12288,0.5007786750793457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,51200,2.407398478190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,51200,2.154375457763672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,12288,0.6711093266805013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,10240,0.4894709269205729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,51200,2.644905598958333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,10240,0.5559605280558269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,8192,0.37647892634073893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,8192,0.4412533442179362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,7168,0.315283203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,7168,0.3882357279459635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,6144,0.2637098630269369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,65536,3.3383967081705728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,6144,0.34144318898518883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,5120,0.22157546679178872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,5120,0.2888319969177246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,16384,0.6562986373901367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,4096,0.18599467277526854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,65536,2.7950347900390624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,4096,0.24775147438049316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,3584,0.16296854019165039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,12288,0.4811680157979329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,3584,0.2111562728881836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,3072,0.13434240023295085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,10240,0.4106485366821289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,3072,0.18338027000427246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,3072,0.14203839302062987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,2560,0.11377493540445964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,8192,0.33141441345214845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,2560,0.1554144064585368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,2048,0.09285439650217692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,2048,0.1288597345352173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,7168,0.28644800186157227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,1536,0.07201600074768066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,1536,0.0991861343383789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,1536,0.08106986681620279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,1024,0.05117119948069254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,1024,0.07386879920959473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,6144,0.24982080459594727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,768,0.041637333234151204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,768,0.06042240063349406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,5120,0.21483093897501626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,512,0.03236053387324016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,512,0.049287466208140056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,4096,0.17531199455261232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,256,0.02219520012537638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,3584,0.15407039324442545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,256,0.03699093262354533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,128,0.01691733400026957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,128,0.031116799513498945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,2560,0.11756373246510823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,64,0.014793599645296732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,64,0.03068586587905884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,2048,0.09785172939300538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,32,0.015427199999491372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,32,0.03035306731859843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,1024,0.06166826486587525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,768,0.05256213347117106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,512,0.04276693264643351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,65536,2.298602549235026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,256,0.0334666649500529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,51200,1.7608373006184894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,128,0.03001813292503357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,16384,0.5313482602437337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,65536,2.9792096455891928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,16384,0.7485557556152344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,51200,2.322989908854167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,10240,0.3140618642171224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,12288,0.5877674738566081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,12288,0.408676274617513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,8192,0.25693012873331705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,10240,0.48123518625895184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,8192,0.38998613357543943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,7168,0.2249834696451823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,7168,0.3451306660970052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,12288,0.37886187235514324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,6144,0.19494080543518066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,6144,0.2218250592549642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,6144,0.29937493006388344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,5120,0.16407893498738607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,5120,0.25268160502115883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,4096,0.13179946740468343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,65536,2.2539019266764324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,4096,0.2070751984914144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,3584,0.11640640099843343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,16384,0.5460320154825846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,3584,0.18421653111775715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,3072,0.10124159653981527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,3584,0.1377994696299235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,3072,0.16129066149393717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,51200,1.7843711853027344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,2560,0.08587733109792074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,2560,0.15200319290161132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,10240,0.34253654479980467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,2048,0.07918612957000733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,2048,0.11459840138753255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,1536,0.054039466381073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,8192,0.2796863873799642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,1536,0.0893994649251302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,1024,0.03965333302815755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,7168,0.24460372924804688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,5120,0.18163199424743653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,768,0.03266666730244954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,768,0.05514346758524576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,768,0.044284800688425704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,512,0.02542933424313863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,4096,0.1474239985148112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,512,0.04379946788152059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,256,0.017767467101415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,1024,0.0655893325805664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,256,0.032178133726119995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,128,0.013656533757845559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,128,0.02923840085665385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,128,0.024338134129842124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,64,0.011982933680216471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,64,0.0288917342821757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,32,0.012468266487121581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,3072,0.11728106339772541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,32,0.028163200616836546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,2560,0.09924373626708985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,2048,0.0837557315826416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,1536,0.06643199920654297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,1024,0.05187840064366659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,65536,1.4951104482014974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,51200,1.089959462483724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,16384,0.37225281397501625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,65536,1.8054986317952473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,16384,0.7139253616333008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,12288,0.26134613355000813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,512,0.0353770653406779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,65536,2.6444107055664063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,256,0.027540266513824463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,10240,0.21084799766540527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,8192,0.17021439870198568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,12288,0.5186026573181153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,51200,2.064557902018229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,10240,0.42532908121744795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,7168,0.15042452812194823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,8192,0.34888639450073244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,7168,0.30244054794311526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,7168,0.2047562599182129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,6144,0.13000960350036622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,6144,0.17221226692199706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,6144,0.2617418607076009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,5120,0.10957012971242268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,5120,0.2204298655192057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,4096,0.08732693195343018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,4096,0.1807029406229655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,16384,0.435700257619222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,3584,0.07847359975179037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,51200,1.3518773396809896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,3584,0.16105492909749347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,3072,0.06802026430765787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,3072,0.14162346522013347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,12288,0.33009068171183265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,3072,0.09332266648610434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,2560,0.05705600182215372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,2560,0.07832533518473307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,2560,0.12204373677571614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,2048,0.0470794677734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,2048,0.10130986372629802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,1536,0.03680959939956665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,1536,0.07813440163930258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,1024,0.025967999299367266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,5120,0.14534613291422527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,1024,0.055094401041666664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,1024,0.04071360031763713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,768,0.021222400665283202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,10240,0.274671999613444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,768,0.044232531388600664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,768,0.03437013228734334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,512,0.015980799992879234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,256,0.01129813293615977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,512,0.03408853212992351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,256,0.024979199965794882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,128,0.00877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,4096,0.11974399884541828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,128,0.021963733434677123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,128,0.01807360053062439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,64,0.007567999760309856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,32,0.00803413341442744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,64,0.021925334135691324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,32,0.02113706668217977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,3584,0.10672319730122884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,65536,1.14682985941569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,8192,0.2290741284688314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,2048,0.06602240006128947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,51200,0.9325354894002279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,16384,0.30712639490763344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,65536,2.4693087259928386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,65536,1.5569226582845053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,512,0.02674986720085144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,12288,0.19837439854939778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,256,0.020520534118016562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,16384,0.38261121114095054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,16384,0.6509930928548177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,10240,0.16684800783793133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,51200,1.9338581085205078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,8192,0.13445653915405273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,10240,0.397653325398763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,12288,0.4758133252461751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,1536,0.05370239814122518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,7168,0.11632426579793294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,8192,0.32300586700439454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,6144,0.10023146470387775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,7168,0.2822485287984212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,7168,0.17698559761047364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,6144,0.15170879364013673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,6144,0.24714879989624022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,5120,0.20721279780069984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,4096,0.07048746744791666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,4096,0.17058879534403484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,3584,0.0613696018854777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,3584,0.15155946413675944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,5120,0.08544960021972656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,3072,0.05316799879074097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,3072,0.1335253397623698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,51200,1.2018367767333984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,12288,0.2939338684082031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,10240,0.24561600685119628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,2560,0.11510079701741535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,2560,0.06955946286519368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,2048,0.036901334921518966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,8192,0.20044479370117188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,2048,0.09426026344299317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,1536,0.02876053253809611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,1536,0.07195946375528971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,1024,0.02060906688372294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,1024,0.04979413350423177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,5120,0.12818559805552165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,768,0.016615466276804606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,768,0.038523733615875244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,4096,0.1070090691248576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,512,0.01306773324807485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,512,0.030689066648483275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,3584,0.09367146492004394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,2560,0.04559893210728963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,256,0.009145599603652955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,256,0.02336639960606893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,256,0.017437867323557534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,128,0.007349333167076111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,128,0.020707199970881142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,3072,0.08161173661549886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,64,0.006415999929110209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,64,0.019883733987808228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,32,0.006567466755708058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,32,0.01954666574796041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,2048,0.05792320171991984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,65536,0.7152138392130534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,1536,0.046513064702351885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,1024,0.03529599905014038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,768,0.028729599714279175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,51200,0.540451176961263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,16384,0.20249493916829428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,16384,0.6302954355875652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,128,0.015121066570281982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,12288,0.13229226271311442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,51200,1.7930826822916668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,12288,0.256874672571818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,512,0.022871466477711995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,12288,0.43914133707682296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,10240,0.10980373223622639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,65536,2.2944981892903646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,8192,0.08900372982025147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,7168,0.0784224033355713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,10240,0.3690431912740072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,8192,0.29910294214884436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,6144,0.06754666964213053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,7168,0.2619061311086019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,6144,0.2289973258972168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,5120,0.056934400399525964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,5120,0.19390932718912762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,65536,1.3093429565429688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,51200,1.011469841003418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,4096,0.04745066563288371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,4096,0.1608565330505371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,3584,0.041059199968973795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,3584,0.1420512040456136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,16384,0.33967787424723306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,3072,0.03638186852137248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,3584,0.08100266456604004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,3072,0.12425599892934162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,2560,0.03100266655286153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,10240,0.21605866750081382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,8192,0.17548906008402507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,2560,0.10640426476796468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,2048,0.02573440074920654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,7168,0.15435093243916828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,2048,0.08695039749145508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,1536,0.019386667013168334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,1536,0.0653983990351359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,1024,0.014751999576886495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,6144,0.13323520024617513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,1024,0.04382186730702718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,768,0.012217600146929424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,768,0.03367786804835002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,5120,0.1117674668629964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,512,0.009776000181833904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,512,0.027050666014353436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,4096,0.0924618641535441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,256,0.0075434664885203045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,256,0.021977599461873373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,3072,0.07103040218353271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,128,0.006311466793219249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,128,0.02003093361854553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,128,0.012176000078519185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,64,0.00555733342965444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,2560,0.05998080174128214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,64,0.01873813271522522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,32,0.005692799886067709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,32,0.01832533280054728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,2048,0.050018131732940674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,1536,0.04001493453979492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,65536,0.4196181297302246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,1024,0.029495465755462646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,51200,0.33425493240356446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,16384,0.10578666528065998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,512,0.01906026601791382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,65536,1.1083413441975911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,12288,0.08234133720397949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,16384,0.554916254679362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,12288,0.4175903956095378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,10240,0.06812266508738199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,51200,1.6854559580485027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,768,0.024132267634073893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,65536,2.145981852213542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,8192,0.055644798278808597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,10240,0.3517439842224121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,256,0.014266666769981385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,7168,0.049352534612019855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,8192,0.1530794620513916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,8192,0.28255252838134765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,6144,0.04381119807561239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,7168,0.25060799916585286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,6144,0.21690452893575035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,5120,0.037647998332977294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,5120,0.18330346743265788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,4096,0.03069546620051066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,16384,0.2947253227233887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,4096,0.1498794714609782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,51200,0.8712885538736979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,3584,0.028689066569010418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,12288,0.22655146916707358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,3584,0.13280853430430095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,3072,0.02491626739501953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,3072,0.11533652941385905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,10240,0.18933226267496744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,2560,0.021257599194844566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,7168,0.13432319959004718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,2560,0.09882240295410157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,2048,0.01404159963130951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,2560,0.05078826745351156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,6144,0.11474773089090984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,1536,0.011171199878056844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,2048,0.08009706338246664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,5120,0.09733546574910482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,1536,0.05792853434880575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,1024,0.008686932921409606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,1024,0.03735893170038859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,1024,0.0232586661974589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,768,0.029971200227737426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,512,0.006263466676076253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,512,0.025037866830825806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,1536,0.03309546709060669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,256,0.005278933544953664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,768,0.007410133381684621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,4096,0.07826879819234213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,128,0.004609066744645437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,128,0.018029866615931193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,128,0.00913813312848409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,64,0.004148266712824504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,64,0.01725546717643738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,32,0.0044383997718493145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,32,0.016875733931859337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,3584,0.06986560026804606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,3072,0.0598090648651123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,65536,0.41727253595987956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,256,0.020667733748753865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,51200,0.32947626113891604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,2048,0.04228053490320842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,16384,0.10836053689320882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,768,0.01918826699256897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,16384,0.5446240107218425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,12288,0.07438613573710123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,512,0.015201066931088766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,65536,2.1454836527506513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,12288,0.41174081166585286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,10240,0.06523946523666382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,51200,1.6720383961995442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,10240,0.3458826700846354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,256,0.01111253301302592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,8192,0.05072000026702881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,7168,0.04485973517100016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,8192,0.27930240631103515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,7168,0.24608000119527182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,6144,0.0399829347928365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,6144,0.21272212664286294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,16384,0.2848885218302409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,51200,0.847540283203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,5120,0.03380053440729777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,65536,1.07652587890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,4096,0.03187519907951355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,5120,0.1805408000946045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,4096,0.14676586786905926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,3584,0.024884267648061117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,12288,0.21421119372049965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,3584,0.13001173337300617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,3072,0.02221119999885559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,3072,0.1126794656117757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,10240,0.18128639856974285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,2560,0.017374932765960693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,2560,0.09566400051116944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,8192,0.14536852836608888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,2048,0.009217066566149394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,2048,0.07519146601359049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,7168,0.12821653683980305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,1536,0.007643733421961467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,1536,0.054426666100819907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,6144,0.11105279922485352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,1024,0.006106666723887126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,1024,0.034227200349171955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,5120,0.09303253491719564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,768,0.005459199845790863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,4096,0.07452267011006673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,768,0.028188800811767577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,512,0.004746666550636292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,512,0.02359893321990967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,3072,0.05638933181762695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,256,0.00412266676624616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,256,0.01880000034968058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,2048,0.03869119882583618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,128,0.0036447999378045404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,1024,0.02106346686681112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,128,0.017449599504470826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,64,0.003386666625738144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,64,0.01667413314183553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,32,0.003688533355792364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,3584,0.06595413287480673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,32,0.016670932372411094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,65536,0.3876288096110026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,2560,0.04825813372929891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,51200,0.3080544153849284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,1536,0.030462932586669923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,16384,0.09470933278401693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,16384,0.5412181218465169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,12288,0.07181759675343832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,65536,2.1324437459309893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,10240,0.06281280120213827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,12288,0.4108917236328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,51200,1.66779301961263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,8192,0.049311999479929605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,7168,0.0435754656791687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,10240,0.3432959874471029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,6144,0.0382912000020345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,8192,0.27752958933512367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,7168,0.24531413714090983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,5120,0.03267733256022136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,4096,0.026846933364868163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,6144,0.21185386975606285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,5120,0.17944854100545246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,3584,0.023860265811284385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,3072,0.02099519968032837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,4096,0.1457013289133708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,2560,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,3584,0.1289258639017741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,2048,0.009469866752624512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,3072,0.11166720390319824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,2560,0.09490346908569336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,1536,0.007977599898974102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,1024,0.006281599899133046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,2048,0.07520853678385417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,768,0.005414400001366933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,1024,0.03409066597620646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,512,0.004548266530036926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,1536,0.053668268521626795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,768,0.02804159919420878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,128,0.003373866776625315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,512,0.02434239983558655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,256,0.018870399395624796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,64,0.0032287999987602235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,512,0.013621333241462707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,64,0.016411733627319337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,32,0.01571626663208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,768,0.01729066570599874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,128,0.007940266529719036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,256,0.009848533074061076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,65536,0.37784748077392577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,16384,0.09348693688710531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,51200,0.2992266654968262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,12288,0.0714090665181478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,10240,0.05973226626714071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,16384,0.540554682413737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,12288,0.41039358774820967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,8192,0.04881173372268677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,10240,0.34312426249186195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,256,0.0038218667109807336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,7168,0.04322559833526611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,6144,0.037725865840911865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,128,0.017768534024556477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,5120,0.031905066967010495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,8192,0.2778271993001302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,4096,0.026279467344284057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,7168,0.24417920112609864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,3584,0.023439999421437582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,3072,0.020618667205174766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,5120,0.17906559308369954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,4096,0.14505066871643066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,3584,0.1282549301783244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,2560,0.013844266533851624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,2048,0.00857919951279958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,1536,0.007269333302974701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,2560,0.09391252994537354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,2048,0.07444799741109212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,1536,0.05228586594263712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,1024,0.03232746720314026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,768,0.004956800242265066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,512,0.004211199780305227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,768,0.0272053341070811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,51200,1.6630176544189452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,256,0.0035690667728583017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,128,0.0032170665760835014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,65536,2.136189778645833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,256,0.018577067057291667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,128,0.016436266899108886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,32,0.003070933371782303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,64,0.0029781334102153777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,64,0.01600213348865509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,32,0.01543786625067393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,32,0.0032927999893824257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,6144,0.21046080589294433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,3072,0.11156906286875408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,1024,0.005496533215045929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,512,0.022725333770116173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,12288,5.01893310546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,16384,6.535251363118489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,16384,9.121485392252605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,12288,9.276607259114583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,16384,12.75828348795573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,10240,4.402486165364584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,10240,8.736775716145832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,12288,6.686357116699218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,10240,5.632916259765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,8192,3.471798451741537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,8192,6.497159322102864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,7168,2.9817738850911457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,51200,22.863401285807292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,7168,5.462069193522135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,8192,4.352339172363282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,6144,2.6157012939453126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,6144,4.837231953938803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,6144,3.4303690592447915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,5120,2.1384841918945314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,7168,3.8716064453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,5120,3.996403249104818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,4096,1.8770688374837239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,3584,1.5409749348958335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,4096,3.413312021891276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,3584,2.7669441223144533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,5120,2.692729695638021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,3072,1.4616597493489585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,3072,2.982349904378255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,4096,2.2305951436360676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,2560,1.180846913655599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,2560,1.9863306681315105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,2048,0.9466538747151694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,2048,1.635696029663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,3584,2.0808778127034504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,2048,1.2109503428141275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,3072,1.6702336629231769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,1536,0.7476128260294597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,1536,1.3669087727864584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,1024,0.5476992289225261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,1024,0.8268778483072916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,768,0.6812320073445638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,768,0.4356682777404785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,512,0.31217705408732094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,512,0.46746133168538406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,2560,1.3814250946044921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,256,0.30446720123291016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,256,0.2653461297353109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,512,0.5206709225972493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,128,0.24644692738850912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,128,0.24396905899047852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,64,0.23018347422281898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,128,0.4048490524291992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,64,0.2384554704030355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,32,0.22465920448303223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,65536,32,0.2318570613861084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,1536,0.9201610565185547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,1024,0.6905077616373698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,768,0.6169301350911458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,65536,51200,40.34878743489583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,256,0.4147072156270345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,65536,51200,40.45301513671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,16384,5.748536682128906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,16384,9.809861246744791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,51200,17.882332356770835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,12288,8.559780883789063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,16384,7.314491780598958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,65536,21.512896728515624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,12288,4.009037780761719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,10240,3.4811648050944006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,10240,6.505797322591145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,12288,5.239802551269531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,10240,4.320320129394531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,8192,2.863480631510417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,8192,5.228522745768229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,51200,31.853200276692707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,7168,2.384232584635417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,7168,4.352850341796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,51200,33.36942545572917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,8192,3.407283274332682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,6144,2.042620849609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,6144,3.767308807373047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,7168,2.9506912231445312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,5120,1.7095007578531902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,5120,3.363396199544271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,4096,1.3552244822184245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,4096,2.471526336669922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,4096,1.803763198852539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,3584,1.2486559549967446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,65536,41.20687255859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,6144,2.525030517578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,3584,2.1967742919921873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,5120,2.105366388956706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,3072,1.07945925394694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,2560,0.8880160013834635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,2560,1.5543360392252603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,2048,1.3150943756103515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,2048,0.7543274561564128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,3072,1.8674592336018878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,1536,0.5883242925008138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,1536,0.7103541056315105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,1536,0.9567605336507162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,1024,0.41538025538126633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,1024,0.6550079981486003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,3584,1.5138207753499349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,768,0.5074144045511881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,768,0.34004265467325845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,768,0.5454922358194987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,512,0.3548970540364583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,512,0.24067625999450684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,2560,1.0815210978190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,256,0.2150474707285563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,256,0.21374826431274413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,128,0.1969696044921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,128,0.197323735555013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,2048,0.9145589192708334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,65536,44.32292073567708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,64,0.18302079836527507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,51200,32,0.17900479634602864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,3072,1.319986089070638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,64,0.22155839602152505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,1024,0.5302890777587891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,51200,32,0.1894015947977702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,512,0.3972810745239258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,256,0.33225920995076497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,51200,128,0.31071360905965173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,16384,1.880413818359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,16384,3.11627197265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,51200,5.700440470377604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,12288,2.313685353597005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,65536,7.736367797851562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,12288,1.3722677866617838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,10240,1.1515455881754557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,51200,9.752977498372395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,10240,2.0305503845214843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,16384,2.2069610595703124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,10240,1.402898152669271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,8192,0.9520992279052735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,8192,1.888702901204427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,7168,0.8733493169148764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,7168,1.4887125651041666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,65536,12.912347412109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,12288,1.6782943725585937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,51200,8.714821370442708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,6144,0.7926218668619792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,6144,1.3562836964925131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,5120,0.5819797515869141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,5120,1.053430430094401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,65536,10.903936767578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,4096,0.4756725311279297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,8192,1.0382399876912436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,4096,0.8490901311238608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,3584,0.6964778900146484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,3584,0.44562772115071614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,3584,0.5339893341064453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,3072,0.3782560030619303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,3072,0.631172243754069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,2560,0.32287467320760094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,2560,0.5089322725931804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,7168,0.921833610534668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,2048,0.2737824122111002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,1536,0.3106741269429525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,1536,0.22373973528544105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,6144,0.7797013600667317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,1024,0.23726293245951333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,5120,0.6636512120564778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,1024,0.16709334055582684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,768,0.16666773160298665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,1024,0.21806186040242515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,2048,0.42238934834798175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,512,0.11923627058664958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,768,0.12909546693166096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,512,0.09665599664052328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,4096,0.5428853352864583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,512,0.1449984073638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,256,0.07519253094991049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,256,0.08213973045349121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,128,0.06555519898732504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,128,0.07359786828358969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,64,0.0595146656036377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,64,0.0739039977391561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,16384,32,0.0595583995183309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,3072,0.4143679936726888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,16384,32,0.07391040325164795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,2560,0.35746987660725915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,256,0.11665173371632893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,2048,0.2947701454162598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,1536,0.24281600316365562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,768,0.16642346382141113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,16384,128,0.10678079922993977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,16384,1.4354975382486979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,16384,2.5784459431966145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,51200,4.708858744303385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,65536,5.9108830769856775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,12288,1.8520064036051433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,12288,1.090164311726888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,12288,1.2558420817057292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,10240,0.9436469395955405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,10240,1.5761194864908854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,51200,7.580342610677083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,65536,8.752970377604168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,8192,0.7751583735148112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,16384,1.6686912536621095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,8192,1.2600308736165364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,65536,9.546535237630208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,7168,1.0516639709472657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,7168,0.6654410680135091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,6144,0.9453557332356771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,6144,0.5783957163492839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,51200,6.015425109863282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,5120,0.4719530741373698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,8192,0.8132895787556966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,5120,0.7991296132405599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,4096,0.3742901484171549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,4096,0.6264650980631511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,4096,0.43896001180013017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,10240,1.0341653188069662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,3584,0.4188096046447754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,3584,0.5168298721313477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,3584,0.38930454254150393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,3072,0.36103251775105794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,2560,0.37721598943074547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,3072,0.4821653366088867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,2560,0.2949930508931478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,2048,0.3318581263224284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,2048,0.2435530662536621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,7168,0.6882474899291993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,1536,0.2646666685740153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,1536,0.16026879946390787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,1024,0.15547733306884765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,1024,0.12698453267415363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,6144,0.6065098444620769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,768,0.131877334912618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,768,0.11061973571777343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,512,0.09429226716359457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,5120,0.501853879292806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,512,0.08146026929219564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,256,0.06038506825764974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,256,0.06663039922714234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,3072,0.31989758809407554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,128,0.05057280063629151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,128,0.06032213370005289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,2560,0.2712810516357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,64,0.04538559913635254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,64,0.059266134103139245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,12288,32,0.045602134863535565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,2048,0.2297269344329834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,12288,32,0.0586570660273234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,1536,0.18862293561299642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,1024,0.14568533897399902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,768,0.12847253481547039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,512,0.1069258689880371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,256,0.0889578660329183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,12288,128,0.08166720072428385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,16384,2.0861279805501303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,16384,1.242956797281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,65536,5.048241170247396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,51200,4.017035675048828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,12288,1.5423540751139322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,12288,0.9306208292643229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,51200,6.099791971842448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,10240,1.2248607635498048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,10240,0.7790634791056316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,8192,0.6506389617919922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,65536,7.907235209147136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,8192,1.002122688293457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,16384,1.4515487670898437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,8192,0.714689064025879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,7168,0.5813119888305665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,7168,0.8654570897420248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,7168,0.6879669189453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,6144,0.4849983851114909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,6144,0.793560536702474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,51200,5.185597737630209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,12288,1.0409215927124023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,5120,0.647709846496582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,5120,0.4078826586405436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,5120,0.46324478785196943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,4096,0.5065386772155762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,65536,6.671726989746094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,4096,0.3814922650655111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,3584,0.413648001352946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,3584,0.30696531931559246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,3072,0.2727914810180664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,3072,0.38505067825317385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,2560,0.30018879572550455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,2560,0.22674454053243004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,2560,0.26892480850219724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,2048,0.26989866892496744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,2048,0.1875999927520752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,10240,0.847531763712565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,1536,0.21113279660542808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,1536,0.14949119885762532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,1024,0.14843734105428058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,1024,0.1111029307047526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,768,0.10669013659159343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,1024,0.1477898597717285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,768,0.0922175963719686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,512,0.07850879828135172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,512,0.07188053131103515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,512,0.09262613455454508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,256,0.04997119903564453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,256,0.059653333822886144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,256,0.07571093241373697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,128,0.043612798055013016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,128,0.052492801348368326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,6144,0.5164469401041667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,128,0.07693546613057455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,64,0.03819733460744222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,64,0.05338346560796102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,4096,0.3537952105204264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,10240,32,0.039348268508911134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,10240,32,0.05255786577860514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,3584,0.31130666732788087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,3072,0.27199678421020507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,2048,0.19792319933573405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,1536,0.16150399843851726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,10240,768,0.1105120023091634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,16384,1.5921941121419272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,16384,1.0478293100992837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,51200,3.3043904622395837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,65536,4.173292795817057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,12288,1.234820302327474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,12288,0.8084831873575846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,51200,4.889209493001302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,10240,0.662497075398763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,10240,1.0078954696655273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,10240,0.7011861165364583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,65536,6.1805974324544275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,8192,0.5607615788777669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,65536,5.154798889160157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,8192,0.7366410573323567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,8192,0.5692746480305989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,7168,0.6820437113444011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,7168,0.4799018541971843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,6144,0.4196661313374837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,6144,0.6183039983113606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,16384,1.1335637410481771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,5120,0.34978132247924804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,5120,0.47316694259643555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,51200,3.6150527954101563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,4096,0.43985706965128585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,4096,0.28762772878011067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,4096,0.29702933629353845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,3584,0.2786474545796712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,3072,0.29431041081746423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,3584,0.3645258585611979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,12288,0.8020448048909505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,3072,0.22848000526428222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,3072,0.236679474512736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,2560,0.26214292844136555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,2560,0.18478399912516277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,2048,0.20937387148539224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,2048,0.16648640632629394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,1536,0.16291839281717937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,1536,0.12736746470133464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,1024,0.10707626342773438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,1024,0.09271252950032552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,1024,0.1075594663619995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,768,0.08322453498840332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,768,0.0781493345896403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,7168,0.4711477279663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,512,0.06745173136393229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,6144,0.41087894439697265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,512,0.06635093291600545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,256,0.04269013404846191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,5120,0.3472127914428711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,256,0.05108480056126913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,256,0.06125439802805582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,128,0.034924801190694174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,128,0.047694933414459226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,64,0.03214186628659566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,64,0.046716801325480145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,3584,0.2513706684112549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,8192,32,0.032415999968846636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,8192,32,0.04675626754760742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,2560,0.18792853355407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,2048,0.15869226455688476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,1536,0.12984533309936525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,768,0.08796586990356445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,512,0.07330133120218912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,8192,128,0.05591573317845663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,16384,0.9184437433878581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,65536,3.759388732910156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,16384,1.3958634694417318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,51200,3.0029014587402343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,12288,0.7241354624430338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,51200,4.32790273030599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,12288,1.0376426696777343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,65536,5.432751973470053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,10240,0.8518357594807944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,10240,0.5902922948201497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,8192,0.6729973475138347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,8192,0.5139199892679851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,7168,0.43392000198364256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,7168,0.5994037628173828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,7168,0.43924694061279296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,6144,0.5114293416341146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,6144,0.39471041361490883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,65536,4.631098429361979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,16384,0.9861610412597657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,51200,3.269251251220703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,5120,0.4321109453837077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,5120,0.31975679397583007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,4096,0.2791765213012695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,4096,0.36431681315104164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,4096,0.25625492731730143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,3584,0.30122667948404946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,10240,0.6065568288167318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,3584,0.22132800420125326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,3584,0.24902826944986978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,3072,0.2751434644063314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,3072,0.19385066032409667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,3072,0.2147424062093099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,2560,0.23234987258911133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,2560,0.1649397373199463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,8192,0.4840256055196126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,12288,0.7364650726318359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,2560,0.1817578633626302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,2048,0.1369983990987142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,2048,0.19008639653523762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,1536,0.13167253335316975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,1536,0.11502506732940673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,1024,0.10143040021260578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,1536,0.119977601369222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,1024,0.08384426434834799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,768,0.08167893091837565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,768,0.06996586322784423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,512,0.059195733070373534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,512,0.055409065882364904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,512,0.07420799732208253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,256,0.03611626625061035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,256,0.04867946704228719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,128,0.031666133801142374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,128,0.044293332099914554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,6144,0.3685237248738607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,64,0.029397332668304445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,64,0.043937067190806076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,7168,32,0.029926399389902752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,7168,32,0.04306559960047404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,5120,0.3066165288289388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,2048,0.1444757302602132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,1024,0.09239359696706137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,768,0.08116906483968099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,256,0.05583146810531616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,7168,128,0.051015468438466396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,16384,0.8403680165608725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,16384,1.1937077840169272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,65536,3.4012255350748695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,12288,0.9494944254557292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,12288,0.643445332845052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,65536,3.8118133544921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,51200,2.660265604654948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,51200,2.8556683858235674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,65536,4.798833211263021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,10240,0.6990367889404296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,10240,0.5599583943684896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,51200,3.7846282958984374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,10240,0.554908816019694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,8192,0.445033613840739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,8192,0.6289173126220703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,7168,0.5038069407145183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,7168,0.3896341323852539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,6144,0.42427520751953124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,6144,0.37124481201171877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,5120,0.3568309466044108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,5120,0.30810025533040364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,5120,0.2928522745768229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,4096,0.29896319707234703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,4096,0.24010772705078126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,4096,0.2397066593170166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,3584,0.2697493235270182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,3584,0.2072970708211263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,3072,0.180293337504069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,3072,0.2352447986602783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,16384,0.8617386500040689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,2560,0.204201602935791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,2560,0.16706879933675128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,2048,0.16284906069437663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,2048,0.13122560183207194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,12288,0.6295199712117513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,1536,0.13081599871317545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,1536,0.1152341365814209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,8192,0.4189770698547363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,1024,0.08114133675893148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,7168,0.3748736063639323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,1024,0.0847050666809082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,1024,0.08024213314056397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,768,0.06735893090566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,768,0.06647040049235026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,768,0.07431466579437256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,6144,0.31863253911336265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,512,0.04844799836476644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,512,0.05547093152999878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,512,0.05858879884084066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,256,0.03346133232116699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,256,0.044069333871205645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,128,0.039612801869710286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,3584,0.20037546157836914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,64,0.023887999852498374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,64,0.03933013280232747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,32,0.02459413409233093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,6144,32,0.03906559944152832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,3072,0.17297919591267902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,2560,0.15034987131754557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,6144,128,0.02759679953257243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,2048,0.12580586274464925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,1536,0.10278506278991699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,256,0.04848853349685669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,6144,128,0.044677333037058516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,65536,2.966895039876302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,16384,0.9205216089884439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,51200,2.3052299499511717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,16384,0.7548095703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,51200,3.293598937988281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,65536,4.136730702718099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,12288,0.7115744272867839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,12288,0.5672373453776042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,10240,0.4684170722961426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,10240,0.626574961344401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,8192,0.46416425704956055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,8192,0.3785877227783203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,8192,0.404692268371582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,7168,0.4458186785380046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,7168,0.3352383931477865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,7168,0.3585439999898275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,6144,0.38533973693847656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,12288,0.5631903966267904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,5120,0.33291734059651695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,65536,3.043664042154948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,5120,0.24919360478719077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,4096,0.20217493375142417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,4096,0.27825066248575847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,4096,0.19448213577270507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,3584,0.23464746475219728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,3584,0.18125333786010742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,16384,0.7175039927164714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,6144,0.29061012268066405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,3072,0.18301973342895508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,2560,0.15516266822814942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,3072,0.17961494127909344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,51200,2.39888432820638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,2560,0.13901333808898925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,2560,0.14020907084147136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,2048,0.14991572697957356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,2048,0.11610240141550701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,2048,0.12229119936625163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,10240,0.451474126180013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,1536,0.11344853242238362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,1536,0.09450133641560873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,1024,0.06813546816507975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,768,0.05470293362935384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,1024,0.08132266998291016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,768,0.05831573406855265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,512,0.0412341316541036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,512,0.05087893406550089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,6144,0.2838837305704752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,256,0.02866986592610677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,256,0.03916480143864949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,5120,0.23540800412495932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,128,0.022830933332443237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,128,0.03625919818878174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,3584,0.17345387140909832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,64,0.021125332514444987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,64,0.035964798927307126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,5120,32,0.022392533222834268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,5120,32,0.03586346705754598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,3072,0.1527392069498698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,1536,0.08999040126800537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,1024,0.07114986578623453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,768,0.0624725341796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,512,0.051436801751454674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,256,0.042269865671793624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,5120,128,0.039043200016021726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,16384,0.7639071782430013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,65536,2.5576502482096357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,51200,1.9827477773030597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,51200,2.6256790161132812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,16384,0.6458730697631836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,12288,0.4945066769917806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,12288,0.5468287785847982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,10240,0.4628330548604329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,10240,0.4193386713663737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,8192,0.4163797378540039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,8192,0.3369525273640951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,7168,0.3214613278706869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,65536,3.3627934773763024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,7168,0.3216896057128906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,6144,0.28357013066609704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,6144,0.26819947560628254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,51200,2.032311503092448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,65536,2.677417500813802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,16384,0.6075839996337891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,5120,0.27560853958129883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,5120,0.2561962604522705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,4096,0.21405760447184244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,12288,0.45453653335571287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,4096,0.18944746653238934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,10240,0.3877888043721517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,3584,0.17459306716918946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,3584,0.16227733294169108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,3072,0.1518389383951823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,3072,0.1390165328979492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,2560,0.12277866999308269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,8192,0.30822614034016926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,2560,0.11791679859161378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,7168,0.27198826471964516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,2048,0.10112426280975342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,2048,0.09815999666849771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,6144,0.2340160051981608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,1536,0.07839893500010173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,1536,0.08017813364664714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,1536,0.07586133480072021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,1024,0.05519253412882487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,1024,0.061713067690531406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,5120,0.20177812576293946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,768,0.04561493396759033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,768,0.05271039803822836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,4096,0.1625461260477702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,512,0.03425279855728149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,512,0.04300053517023723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,3584,0.14613653818766276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,256,0.024205867449442545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,3072,0.12749333381652833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,256,0.035420799255371095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,256,0.03394560019175212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,128,0.018981333573659262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,128,0.030935466289520264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,2560,0.11115626494089763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,64,0.016633599996566772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,4096,32,0.01689173380533854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,64,0.03097066680590312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,4096,32,0.03051946759223938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,2048,0.09303359985351563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,1024,0.05903146664301554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,768,0.05185066858927408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,65536,2.3412747701009113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,512,0.04274026552836101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,65536,2.81760737101237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,4096,128,0.03239360054334005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,16384,0.5865119934082031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,16384,0.6649749120076497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,12288,0.4745770772298177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,51200,1.8106751759847004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,10240,0.39694293340047204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,10240,0.3858037312825521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,8192,0.31381654739379883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,51200,2.2482081095377606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,8192,0.31173760096232095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,7168,0.2766304016113281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,12288,0.44708798726399734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,7168,0.27725868225097655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,6144,0.23821867307027184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,6144,0.24159466425577797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,12288,0.4892245292663574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,5120,0.2014143943786621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,5120,0.20420479774475098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,16384,0.6425216039021809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,65536,2.8958836873372396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,4096,0.16508906682332355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,4096,0.18430293401082357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,3584,0.14843093554178874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,3584,0.14915733337402343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,10240,0.421233081817627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,3072,0.12600639661153157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,3072,0.12780693372090657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,6144,0.2440821329752604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,2560,0.10434133211771648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,2560,0.10841600100199382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,7168,0.29241600036621096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,2048,0.08580693403879801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,2048,0.09108479817708334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,51200,2.142736053466797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,1536,0.07581973075866699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,1536,0.07881920337677002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,8192,0.32365760803222654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,1024,0.0482965350151062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,1024,0.05731093486150106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,3072,0.12155199845631917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,768,0.03952533404032389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,768,0.04872746864954631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,4096,0.16630080540974934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,512,0.03007253408432007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,1536,0.06703573067982992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,512,0.040677332878112794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,256,0.021220266819000244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,256,0.03242666721343994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,5120,0.20348800023396812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,128,0.01639573375384013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,128,0.02842346628506978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,2560,0.10822079976399739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,64,0.014173866311709086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,64,0.028474666674931842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3584,32,0.014265599846839904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3584,32,0.028654932975769043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,3584,0.14342506726582843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,1024,0.05312960147857666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,2048,0.08818133672078451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,65536,2.151684315999349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,65536,2.4513206481933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,51200,1.9144864400227863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,16384,0.6206261316935222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,51200,1.694372304280599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,16384,0.5614869435628255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,256,0.02754773298899333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,12288,0.41611200968424483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,128,0.02295359969139099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,512,0.03365866740544637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,10240,0.3497887929280599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3584,768,0.041971198717753094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,10240,0.380297597249349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,8192,0.28226133982340496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,8192,0.29801387786865235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,12288,0.40469973882039384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,7168,0.24085653622945152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,7168,0.2518976052602132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,6144,0.210917329788208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,6144,0.22005012830098472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,65536,2.5980852762858073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,5120,0.19268693923950195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,5120,0.1991477330525716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,12288,0.439957332611084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,4096,0.14179840087890624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,4096,0.151638396581014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,16384,0.590130106608073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,3584,0.12545599937438964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,3584,0.1380661328633626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,51200,1.935597864786784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,3072,0.1082528034845988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,10240,0.3738474527994792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,3072,0.11841599941253662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,2560,0.09171306292215983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,2560,0.10138560136159261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,6144,0.2179488023122152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,7168,0.259498659769694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,2048,0.07671360174814859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,2048,0.0852725346883138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,8192,0.3011232058207194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,1536,0.05902080138524374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,1536,0.07055892944335937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,1024,0.043355735143025716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,1024,0.054472533861796055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,4096,0.154585599899292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,768,0.0350655992825826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,768,0.0458026647567749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,5120,0.18449494043986003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,512,0.02717333237330119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,512,0.0375925342241923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,3584,0.1329248030980428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,256,0.019489065806070963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,3072,0.11417280038197834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,256,0.030859732627868654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,128,0.016100266575813295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,1536,0.06191146771113078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,128,0.027276800076166792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,64,0.013610666990280152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,64,0.028437334299087524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,3072,32,0.013904000322024027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,3072,32,0.027635200818379717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,2560,0.09973226388295492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,1024,0.04844586849212647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,65536,1.9840192159016925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,2048,0.08183680375417074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,65536,1.962942886352539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,51200,1.5034005482991537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,16384,0.4918506622314453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,51200,1.5433119455973308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,16384,0.5242069244384766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,128,0.020567466815312706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,12288,0.3437525431315104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,256,0.02500480016072591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,12288,0.37992747624715173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,512,0.03050559957822164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,10240,0.2887839953104655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,10240,0.31809813181559243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,8192,0.2334709326426188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,8192,0.25913386344909667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,3072,768,0.038585599263509116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,7168,0.20120107332865395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,7168,0.23095253308614097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,6144,0.1752991994222005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,6144,0.20139840443929038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,12288,0.38104321161905924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,65536,2.1303637186686197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,5120,0.14757653872172039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,5120,0.1857098738352458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,16384,0.5122912089029948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,4096,0.12279360294342041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,4096,0.14109973907470702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,3584,0.10687999725341797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,3584,0.12400853633880615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,51200,1.6225546518961589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,3072,0.09922453562418619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,3072,0.11539519627888997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,10240,0.3333141326904297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,2560,0.07927786509195964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,2560,0.09277119636535644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,6144,0.19685440063476561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,8192,0.2616031964619955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,2048,0.06479039986928305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,2048,0.07973546981811523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,1536,0.050853331883748375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,1536,0.06438186566034952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,7168,0.22707306543986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,1024,0.03646080096562703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,1024,0.048951466878255204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,3072,0.10096533298492431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,768,0.02955519954363505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,768,0.04132800102233887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,4096,0.13322880268096923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,512,0.023081600666046143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,512,0.034482133388519284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,5120,0.16353386243184406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,256,0.016778665781021117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,1536,0.05494399865468343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,2560,0.09011092980702719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,128,0.013060266772905985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,128,0.025963733593622845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,3584,0.11664533615112305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,64,0.011396267016728719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,64,0.02515520056088765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2560,32,0.011773866415023804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,32,0.025029333432515462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2560,256,0.02842880090077718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,65536,1.4616981506347657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,2048,0.07200960318247476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,65536,1.7623967488606769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,51200,1.1565685272216797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,51200,1.3796128590901693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,16384,0.35863361358642576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,128,0.0184714674949646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,16384,0.4691914558410645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,256,0.02193386753400167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,12288,0.27062292098999025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,512,0.02717333237330119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,12288,0.3435317357381185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,10240,0.23084799448649088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,10240,0.28952318827311196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,1024,0.04304533402125041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,8192,0.18355414072672527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2560,768,0.034687999884287515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,8192,0.23360853195190429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,7168,0.15781973203023275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,7168,0.20583893458048502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,6144,0.1373162587483724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,6144,0.17910505930582682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,65536,1.7697364807128906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,5120,0.12147626876831055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,5120,0.1636799971262614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,12288,0.32732906341552737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,4096,0.09399360020955404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,4096,0.12554666996002198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,16384,0.43445971806844075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,3584,0.08352426687876383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,6144,0.1643328030904134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,3584,0.11216213703155517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,3072,0.07230079968770345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,10240,0.2805760065714518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,7168,0.19418986638387042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,51200,1.3502527872721353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,2560,0.06035840113957723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,2560,0.09089386463165283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,2048,0.055062401294708255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,2048,0.07287573019663493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,3072,0.09778026739756265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,1536,0.03881706794102986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,1536,0.05468266805013021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,8192,0.22232534090677897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,1024,0.02757013241449992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,1024,0.04153493245442708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,3072,0.08797866503397624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,768,0.022951465845108033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,4096,0.11765653292338055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,768,0.03533866802851359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,512,0.017515732844670614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,512,0.028666667143503827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,5120,0.1390559991200765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,256,0.012140799562136333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,1536,0.047977598508199056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,256,0.023460266987482707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,128,0.009393067161242167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,128,0.024396799008051553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,2560,0.07480746905008952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,64,0.010466133554776508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,64,0.024131200710932412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,3584,0.10188053448994953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,2048,32,0.0107914666334788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,2048,32,0.02419840097427368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,1024,0.03784000078837077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,65536,1.1414549509684244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,2048,0.06291626691818238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,65536,1.5652180989583333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,51200,0.9351445515950522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,16384,0.28309653600056967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,16384,0.42351360321044923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,51200,1.2247071584065756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,12288,0.20215466817220054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,12288,0.3156160036722819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,128,0.01597866714000702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,10240,0.17051199277242024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,256,0.019052799542744955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,512,0.02376000086466471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,10240,0.25785813331604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,8192,0.13915732701619465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,8192,0.20782292683919273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,7168,0.1208725372950236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,2048,768,0.03062186638514201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,7168,0.18472213745117189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,6144,0.10393813451131184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,6144,0.1613525390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,65536,1.470037333170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,5120,0.09002026716868082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,12288,0.2846282641092936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,5120,0.140447998046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,4096,0.07134719689687094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,4096,0.11568960348765056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,16384,0.35578346252441406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,3584,0.06172800064086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,3584,0.10016000270843506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,51200,1.1449461619059245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,3072,0.05713280042012533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,3072,0.09149973392486573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,10240,0.23490880330403646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,2560,0.046431998411814376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,2560,0.07423146565755209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,7168,0.16084906260172527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,8192,0.18116159439086915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,2048,0.037463466326395675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,2048,0.061345068613688145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,1536,0.029470932483673096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,1536,0.04793066581090291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,6144,0.1405845324198405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,1024,0.020836265881856282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,1024,0.035369598865509035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,3072,0.07107733090718588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,768,0.017399466037750243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,768,0.029950932661692305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,4096,0.09890986283620198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,512,0.013153066237767538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,512,0.025990400711695356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,5120,0.11947946548461914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,256,0.009271466732025146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,3584,0.08158613046010335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,256,0.021811199188232423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,128,0.007561600208282471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,128,0.01993173360824585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,1536,0.03901546796162923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,64,0.006764799853165944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,64,0.019578667481740315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,2560,0.062393601735432944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1536,32,0.007113599777221679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1536,32,0.019466666380564372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,65536,0.7416309356689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,65536,1.3932640075683593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,2048,0.04945493141810099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,51200,0.5658581415812175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,1024,0.029445334275563555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,16384,0.1822538693745931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,16384,0.36084372202555337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,51200,1.0934069315592447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,12288,0.13655254046122234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,12288,0.27181866963704426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,512,0.019057067235310872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,10240,0.11512640317281086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,128,0.013314132889111837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,256,0.015052800377209982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,8192,0.09105280240376791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,10240,0.2301472028096517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,8192,0.18718400001525878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1536,768,0.024272000789642333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,7168,0.08056106567382812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,7168,0.16750720342000325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,6144,0.07004799842834472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,6144,0.14649492899576824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,65536,1.2413472493489583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,5120,0.06285760005315146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,5120,0.13158186276753742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,12288,0.23220799763997396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,4096,0.04807146787643433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,4096,0.1029311974843343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,16384,0.30911146799723305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,3584,0.043112532297770186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,3584,0.08974826335906982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,51200,0.948477872212728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,3072,0.03805546760559082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,3072,0.08096960385640463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,6144,0.11906987031300861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,2560,0.03143786589304606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,2560,0.06514240105946859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,10240,0.19689706166585286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,2048,0.025641600290934246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,2048,0.05212586720784505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,7168,0.13944533665974934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,1536,0.020844799280166627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,1536,0.040462934970855714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,8192,0.15934613545735676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,1024,0.015254400173823037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,1024,0.030214399099349976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,4096,0.0836896022160848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,768,0.012619733810424805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,768,0.02627413272857666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,5120,0.10066986878712972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,512,0.010397866368293762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,512,0.02372480034828186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,1536,0.03325120011965434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,256,0.00767146646976471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,3072,0.06238293250401815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,256,0.020482132832209267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,3584,0.0718826691309611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,128,0.006494933366775512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,128,0.0189301331837972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,2560,0.05359679857889811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,64,0.005756799876689911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,64,0.018967467546463012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,1024,32,0.006088533500830332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,1024,32,0.018477867046991982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,65536,0.5339808146158854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,1024,0.02469866673151652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,2048,0.042662398020426436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,51200,0.41907094319661453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,65536,1.3216512044270834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,51200,1.0119061152140298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,16384,0.136135466893514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,16384,0.3344501177469889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,12288,0.10151360034942628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,12288,0.25421120325724283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,128,0.010899200042088827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,10240,0.0871445337931315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,256,0.012345600128173827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,512,0.01592639982700348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,10240,0.21590612729390463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,8192,0.07120532989501953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,1024,768,0.019822933276494346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,8192,0.17595307032267254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,7168,0.06132266521453857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,7168,0.1559658686319987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,6144,0.053658668200174967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,6144,0.13613333702087402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,65536,1.0031722386678061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,5120,0.04533546765645345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,5120,0.1195029338200887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,12288,0.1890570640563965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,4096,0.03607253233591716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,4096,0.09606613318125407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,16384,0.24956587155659996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,3584,0.03237653374671936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,3584,0.08481386502583822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,51200,0.7617162704467774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,6144,0.09793813228607177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,3072,0.028336000442504884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,3072,0.07274666627248129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,2560,0.0240831991036733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,2560,0.06023893356323242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,10240,0.16229012807210286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,2048,0.019821866353352865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,2048,0.04782400131225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,7168,0.11535893281300862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,1536,0.01573013365268707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,1536,0.0365503986676534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,8192,0.1309226671854655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,1024,0.011992533008257549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,1024,0.029186133543650312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,3072,0.05125120083491007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,768,0.009985066453615824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,768,0.02518506646156311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,4096,0.06818133195241292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,512,0.008063999811808269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,512,0.022204800446828207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,1536,0.027499733368555705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,256,0.006295466423034668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,256,0.020116267601648967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,5120,0.0820682684580485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,128,0.005340800185998281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,128,0.01772693395614624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,2560,0.0434826652208964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,64,0.004887466629346212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,64,0.017449599504470826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,768,32,0.005138133466243744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,768,32,0.017375999689102174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,65536,0.393068790435791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,3584,0.05830613374710083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,2048,0.03528639872868856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,1024,0.020909865697224937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,51200,0.3106464068094889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,65536,1.2337141672770182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,16384,0.10108373165130616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,16384,0.3144159952799479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,51200,0.9645610809326172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,12288,0.07787626584370931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,12288,0.24230613708496093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,128,0.008781866232554118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,10240,0.0643722653388977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,256,0.010201600193977357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,10240,0.2055957317352295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,8192,0.05209066470464071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,512,0.013522133231163025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,8192,0.1681418736775716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,7168,0.04605013529459635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,7168,0.1490645408630371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,768,768,0.017386666933695474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,6144,0.039572266737620036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,6144,0.13037226994832357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,12288,0.18581013679504393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,5120,0.03439679940541585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,5120,0.11139840284983318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,16384,0.24209493001302085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,65536,0.9788373311360677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,4096,0.02752000093460083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,4096,0.09094613393147787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,51200,0.7482421239217122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,3584,0.02404800057411194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,3584,0.07908799648284912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,10240,0.15323519706726074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,3072,0.021077332894007365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,3072,0.06696639855702718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,6144,0.09355200131734212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,2560,0.01836586594581604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,2560,0.055586131413777676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,7168,0.11006933053334553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,2048,0.015251200397809347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,2048,0.042445866266886394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,8192,0.12737706502278645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,1536,0.012588799993197123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,1536,0.03374613523483276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,1024,0.009713066617647807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,1024,0.027461334069569902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,4096,0.06578239997227987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,768,0.00829013337691625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,5120,0.07975893020629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,768,0.02400853236516317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,512,0.006878933310508728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,512,0.021602133909861244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,3072,0.050063999493916836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,256,0.005454933146635691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,256,0.019013333320617675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,3584,0.058994134267171226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,128,0.005099733173847198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,128,0.017248000701268515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,1536,0.0269375999768575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,64,0.004332800209522247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,64,0.01726186672846476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,512,32,0.0044501334428787235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,512,32,0.016716800133387246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,65536,0.2304405371348063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,2560,0.041725865999857586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,1024,0.019744000832239785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,51200,0.18122132619222003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,2048,0.03504213492075602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,65536,1.154980214436849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,16384,0.07659626801808675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,16384,0.301474126180013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,51200,0.9097909291585287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,12288,0.06021866798400879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,12288,0.23112427393595375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,128,0.008185600241025288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,10240,0.05045226812362671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,256,0.009322667121887207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,10240,0.19769387245178222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,512,0.01274773379166921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,8192,0.04246826569239299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,8192,0.16183573404947918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,7168,0.03673706849416097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,7168,0.14388799667358398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,512,768,0.016711467504501344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,6144,0.031090132395426434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,6144,0.12555519739786786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,65536,0.6502304077148438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,5120,0.02871893246968587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,5120,0.10751360257466633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,12288,0.12874560356140136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,4096,0.02386666735013326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,4096,0.08708906968434652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,16384,0.17151145935058593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,3584,0.019939200083414713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,3584,0.07585386435190836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,51200,0.5135583877563477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,3072,0.017450666427612303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,3072,0.06288959980010986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,10240,0.1096117337544759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,2560,0.015611732999483744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,2560,0.05039786497751871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,6144,0.06779946486155192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,2048,0.013004799683888754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,7168,0.0800277312596639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,2048,0.03951893250147502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,1536,0.010748799641927083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,8192,0.08759360313415528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,1536,0.03219306667645772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,1024,0.007898666461308797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,1024,0.025770666201909383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,4096,0.047076265017191574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,768,0.006697600086530049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,768,0.02299413283665975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,3072,0.03652160167694092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,512,0.0053269331653912864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,512,0.020942932367324828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,5120,0.05664746761322022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,256,0.00450133333603541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,256,0.01752106746037801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,3584,0.04173440138498942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,128,0.003923200070858002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,128,0.01646613379319509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,1536,0.019834667444229126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,64,0.0036042665441830954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,2560,0.03110719919204712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,64,0.016218666235605875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,256,32,0.003602133442958196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,256,32,0.015682133038838704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,65536,0.22002666791280112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,1024,0.015267200271288552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,65536,1.144655990600586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,51200,0.17671573956807454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,2048,0.02570880055427551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,51200,0.898090680440267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,16384,0.06107519865036011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,256,0.00786240001519521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,16384,0.29614079793294273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,12288,0.04217919905980428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,512,0.009940266609191895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,128,0.006861866513888042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,12288,0.22565973599751793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,10240,0.036160000165303546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,10240,0.19039573669433593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,8192,0.029407999912897748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,256,768,0.012552533547083536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,8192,0.15520639419555665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,7168,0.026246400674184163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,7168,0.13705280621846516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,6144,0.023434666792551677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,6144,0.11871360143025715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,65536,0.6407605489095052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,5120,0.0200437327226003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,5120,0.10206826527913411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,12288,0.12413547039031983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,4096,0.01229866643746694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,4096,0.0812981367111206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,16384,0.16581652959187826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,51200,0.5126901308695475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,3584,0.010969600081443787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,3584,0.07121813297271729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,3072,0.010358400146166484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,3072,0.05908906857172648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,10240,0.10509653091430664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,2560,0.008845866719881693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,6144,0.0651573340098063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,2560,0.04650346835454305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,2048,0.007686399916807811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,2048,0.036442665259043376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,7168,0.0751850684483846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,1536,0.0070816000302632645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,1536,0.029674667119979858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,8192,0.08475519816080729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,1024,0.005705599983533224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,1024,0.024474666515986124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,5120,0.0548799991607666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,768,0.005141333242257436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,768,0.02185279925664266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,3072,0.035998932520548504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,4096,0.045465600490570066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,1536,0.018720000982284546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,512,0.019364267587661743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,256,0.0037237333754698435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,256,0.016851200660069784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,3584,0.04103999932607015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,2560,0.028969599803288774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,128,0.0033781332274278007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,128,0.016263467073440552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,64,0.0032074667513370516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,64,0.015774933497111003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,32,0.003257599969704946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,128,32,0.015482667088508605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,2048,0.023596799373626708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,1024,0.014101333419481912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,65536,0.20867519378662108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,51200,0.16589439709981282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,16384,0.05726933479309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,16384,0.29366718928019203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,12288,0.04486720164616902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,51200,0.8947519938151041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,65536,1.141852823893229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,10240,0.03990506728490194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,8192,0.02836479942003886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,12288,0.2241567929585775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,7168,0.025161600112915038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,10240,0.189738670984904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,8192,0.15421759287516276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,6144,0.0226474662621816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,7168,0.13624213536580404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,5120,0.018156800667444864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,4096,0.01123413344224294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,6144,0.11816639900207519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,5120,0.10000533262888592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,3584,0.009955199559529622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,3072,0.00894719958305359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,4096,0.08102933565775552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,128,512,0.004448000093301137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,2560,0.007645866771539052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,3584,0.06894720395406087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,2048,0.007044266661008198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,1536,0.005913599828879039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,2048,0.036039467652638754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,2560,0.04482560157775879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,1024,0.004671999812126159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,1536,0.029095466931660968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,768,0.00421013335386912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,1024,0.023835732539494833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,512,0.003874133278926214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,768,0.021524266401926676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,256,0.0033439998825391137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,512,0.018962132930755615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,128,0.0031701333820819853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,128,0.015701333681742348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,64,0.0030400000512599947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,64,0.015332266688346863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,64,32,0.0030122667551040648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,32,0.015004799763361613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,65536,0.21200213432312012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,3072,0.05786346594492594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,51200,0.1658954620361328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,64,256,0.01752426624298096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,16384,0.05808746814727783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,65536,1.1425151824951172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,12288,0.04376213153203328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,256,0.007012266914049785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,16384,0.2966858545939127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,128,0.00603413333495458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,512,0.009175466497739156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,10240,0.03775146802266439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,8192,0.030666667222976684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4353,128,768,0.011532800396283467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,51200,0.8953813552856446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,7168,0.025177599986394246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,12288,0.22387305895487467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,5120,0.019195733467737834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,6144,0.02272426684697469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,8192,0.153656530380249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,4096,0.011914666493733723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,7168,0.1362101395924886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,3584,0.010290132959683736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,6144,0.11762453715006511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,5120,0.10012053648630778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,4096,0.08019946416219076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,3584,0.06839253107706705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,2560,0.008130133152008057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,2048,0.007175466914971669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,3072,0.05711040099461874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,2560,0.044649600982666016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,1024,0.004508799811204275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,2048,0.03393599987030029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,1536,0.028424533208211263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,1024,0.023744000991185506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,512,0.003621333340803782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,768,0.021590399742126464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,256,0.003176533430814743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,512,0.019330133994420372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,128,0.0030400000512599947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,256,0.01686613361040751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,64,0.0028736000259717304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,128,0.016297599673271178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,32,0.002855466554562251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,32,0.014855466286341348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,10240,0.18908054033915203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,3072,0.008358400066693623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,1536,0.005640533566474914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4353,32,768,0.004158933212359746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4353,32,64,0.0151936004559199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,12288,5.28895263671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,16384,6.681252034505208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,16384,8.026160176595052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,12288,9.352504475911458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,12288,6.227000427246094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,16384,12.884799194335937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,10240,4.326714579264323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,10240,9.090485636393229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,8192,3.766060892740885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,10240,5.318608093261719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,8192,6.643878173828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,7168,3.119515736897786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,8192,4.083892313639323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,7168,5.572979227701823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,51200,22.6360107421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,7168,3.5189674377441404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,6144,2.595164744059245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,6144,4.836506652832031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,5120,2.2112075805664064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,5120,4.015817515055338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,4096,1.758881123860677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,4096,3.2260821024576822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,4096,2.1214623769124348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,6144,3.1594805399576824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,5120,2.5830304463704428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,3584,1.8607808430989583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,3584,3.1399497985839844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,3072,1.289675776163737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,3072,2.7439478556315104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,3072,1.6126304626464845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,2560,1.236737060546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,2560,2.2209846496582033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,2048,0.9408917109171548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,2048,1.2718527475992838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,2048,1.7580331166585286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,3584,1.948903528849284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,1536,0.7260874430338542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,1536,1.2759872436523438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,1024,0.9064767837524415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,1024,0.553714116414388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,1024,0.7349269231160481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,768,0.7021162668863933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,768,0.470904541015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,512,0.5156288146972656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,512,0.3465322812398275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,2560,1.404696528116862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,256,0.29573974609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,256,0.2584938685099284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,256,0.4241312026977539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,128,0.2373098691304525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,128,0.23505919774373374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,64,0.22011946042378744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,64,0.22913920084635414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,32,0.21525866190592446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,1536,0.9198015848795572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,51200,32.6276611328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,768,0.6066122690836588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,65536,32,0.22532480557759604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,512,0.48727038701375325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,65536,128,0.3909642537434896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,65536,51200,41.62958984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,16384,6.114228312174479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,16384,9.82543436686198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,51200,18.17786865234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,12288,8.30005340576172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,16384,6.588733927408855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,65536,22.055975341796874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,12288,4.1914006551106775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,10240,3.5351285298665367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,10240,4.2114814758300785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,10240,6.462158711751302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,51200,22.099711100260414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,12288,4.953325907389322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,8192,3.1166709899902343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,7168,2.5063636779785154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,8192,5.328373209635417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,8192,3.7354751586914063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,51200,32.01511840820312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,7168,4.729118855794271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,6144,2.037597910563151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,7168,3.021014404296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,6144,2.5552576700846354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,5120,1.8780768076578778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,6144,3.7800320943196617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,5120,3.087603251139323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,5120,2.1053877512613934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,4096,1.448046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,4096,2.564594014485677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,3584,1.1879861195882162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,65536,33.90242513020833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,3584,2.699330139160156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,3584,1.536730702718099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,3072,1.0974868774414062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,3072,1.949355697631836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,2560,0.9839285532633463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,2048,0.7755989074707031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,2560,1.482424545288086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,2048,1.308023452758789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,4096,1.6663093566894531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,1536,0.5971797307332356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,1536,0.9976394653320313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,1536,0.7414026896158854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,1024,0.4302943865458171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,1024,0.695849609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,1024,0.6366293589274089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,768,0.5135050773620605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,65536,41.224979654947916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,768,0.36303466161092124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,512,0.30177599589029946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,512,0.3858165423075358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,256,0.2444437344868978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,128,0.18680639266967775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,256,0.2068544069925944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,256,0.3351093292236328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,512,0.4167946815490723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,128,0.19406186739603679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,64,0.17471466064453126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,51200,32,0.17209919293721515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,64,0.2094495932261149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,2560,1.0110794703165689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,3072,1.1837599436442057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,51200,32,0.18288745880126953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,2048,0.8761738459269205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,768,0.4763008117675781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,51200,128,0.3078890800476074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,16384,1.8615637461344403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,16384,3.21300048828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,51200,6.262143961588541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,65536,7.455120849609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,12288,2.3383265177408856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,12288,1.4488885243733725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,10240,1.30819943745931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,12288,1.5130794525146485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,51200,9.625687662760416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,10240,1.9868757883707684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,16384,2.1834698994954427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,51200,6.564930216471355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,10240,1.3845887502034506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,8192,0.913758913675944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,8192,1.6870399475097657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,8192,1.0275552113850912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,7168,0.8452106475830078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,6144,0.7536341349283855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,6144,1.4119338989257812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,7168,1.4900245666503906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,65536,9.121672566731771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,6144,0.7890058517456054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,65536,13.341207885742188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,5120,0.6481109619140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,5120,0.9914602915445963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,5120,0.7234186808268229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,4096,0.4827168146769206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,4096,0.7995317459106446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,3584,0.45313173929850264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,3584,0.6973194758097331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,4096,0.6587434768676758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,3072,0.602509880065918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,3072,0.3881674766540527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,2560,0.3258634567260742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,2048,0.39462505976359047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,2560,0.4086325327555339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,2048,0.25787199338277184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,1536,0.31064319610595703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,1536,0.20283625920613607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,1536,0.2590474605560303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,1024,0.20425492922465005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,7168,0.8930165608723959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,1024,0.1563434600830078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,1024,0.20560213724772133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,768,0.19900159835815429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,768,0.1263487974802653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,512,0.13813759485880533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,512,0.09903146425882975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,256,0.08859732945760092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,256,0.08141547044118246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,2560,0.5404341379801433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,128,0.06332159837086995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,128,0.07462293306986491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,128,0.10118827025095622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,64,0.058441599210103355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,64,0.07219093640645345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,2048,0.31907841364542644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,16384,32,0.05842666625976563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,16384,32,0.07212053140004476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,3584,0.4510656038920085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,3072,0.3980127970377604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,768,0.16049706141153972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,512,0.13236052989959718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,16384,256,0.11167679627736408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,16384,1.5300565083821616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,16384,2.5704991658528646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,51200,4.7610529581705725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,65536,6.106083170572917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,12288,1.8277056376139325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,12288,1.1160896301269532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,12288,1.1984864552815755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,51200,7.546655782063802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,10240,0.9561728159586588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,10240,1.5285631815592446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,8192,0.7963935852050781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,8192,1.2155733744303385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,16384,1.6343690236409505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,65536,9.872632853190105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,65536,6.923472086588542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,7168,1.130838394165039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,7168,0.6691680272420247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,6144,0.8773183822631836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,6144,0.6097728093465169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,6144,0.6152352015177409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,51200,4.981920878092447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,5120,0.7526485443115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,5120,0.4785610516866048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,5120,0.4954122543334961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,4096,0.40672000249226886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,4096,0.6170656204223632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,3584,0.3612863858540853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,3584,0.5395754496256511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,10240,0.9972693125406901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,3072,0.48479359944661454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,3072,0.3065055847167969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,2560,0.41913814544677735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,2560,0.2830688158671061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,2560,0.2791253407796224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,2048,0.3315114657084147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,2048,0.22637866338094076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,2048,0.23862719535827637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,8192,0.7542901357014974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,1536,0.17287467320760092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,1024,0.17052480379740398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,1024,0.12617493470509847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,768,0.12384320100148519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,768,0.10857919851938884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,7168,0.683193588256836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,768,0.138265593846639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,512,0.10121066570281982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,512,0.08663360277811685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,256,0.059577600161234534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,256,0.06851413249969482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,1536,0.2531189282735189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,128,0.049244801203409835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,4096,0.3864202817281087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,128,0.05931200186411539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,3584,0.3481642723083496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,64,0.04500480095545451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,64,0.05903040170669556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,3072,0.30335359573364257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,12288,32,0.0459445317586263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,12288,32,0.05954879919687907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,1536,0.1810976028442383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,1024,0.14129600524902344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,512,0.10343573093414307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,256,0.08589653174082437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,12288,128,0.07953706582387289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,16384,1.2718282063802084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,16384,2.0908362070719404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,65536,5.132494099934896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,51200,4.212066141764323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,51200,4.190177154541016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,12288,1.6454784393310546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,65536,5.660105387369791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,12288,0.9450346628824869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,12288,1.0355050404866537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,51200,6.269034830729167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,10240,0.8360917409261068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,10240,0.8642592112223306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,8192,1.0478901545206705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,10240,1.2910719553629557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,8192,0.7034784317016601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,8192,0.6448853174845378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,65536,8.046563212076823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,7168,0.6079210917154948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,7168,0.9175200144449869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,6144,0.779262924194336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,6144,0.5290602684020996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,5120,0.43410666783650714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,16384,1.3333343505859374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,5120,0.6181653340657551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,4096,0.35818560918172204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,4096,0.3493184089660645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,4096,0.5346239725748698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,3584,0.3068127950032552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,3584,0.4892586708068848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,3584,0.31116161346435545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,3072,0.39384638468424477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,3072,0.29323838551839193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,2560,0.2408672014872233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,2048,0.2733685175577799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,2048,0.193995730082194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,2048,0.21410773595174154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,1536,0.19888213475545247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,1536,0.15419306755065917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,1024,0.13747626940409344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,1024,0.11173866589864094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,2560,0.3525024096171061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,768,0.10183359781901043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,768,0.09026347001393636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,7168,0.5674378712972005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,512,0.08724053700764975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,512,0.07889493306477866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,6144,0.4777194658915202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,256,0.047326934337615964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,5120,0.412282657623291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,256,0.07351253032684327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,256,0.0735754648844401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,128,0.0580842653910319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,128,0.07164586385091146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,64,0.05743786493937174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,64,0.07147626876831055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,10240,32,0.05797119935353597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,3072,0.2588106632232666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,2560,0.22500160535176597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,10240,32,0.07161813577016195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,1536,0.15682026545206706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,1024,0.12039146423339844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,768,0.1061557372411092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,512,0.0887178659439087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,10240,128,0.06755200227101645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,16384,1.0512287775675455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,16384,1.6427178700764973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,65536,4.262302907307943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,51200,3.383349355061849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,65536,4.646647644042969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,16384,1.140988794962565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,12288,0.7918357213338216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,51200,3.422700754801432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,12288,1.2709546407063803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,10240,1.0217034657796225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,10240,0.7081546783447266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,8192,0.5808095932006836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,10240,0.7064042409261068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,8192,0.8117578506469727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,65536,6.367271423339844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,7168,0.4853952089945476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,7168,0.5241536140441895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,7168,0.7049866358439127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,6144,0.6153290430704753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,6144,0.41260480880737305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,6144,0.4321141242980957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,5120,0.3783562660217285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,5120,0.5321322758992513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,4096,0.28639361063639324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,4096,0.29994026819864905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,4096,0.40709333419799804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,51200,5.119813537597656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,3584,0.2582026640574137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,3584,0.3386901219685873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,3072,0.30014934539794924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,3072,0.21973439852396645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,2560,0.2516416072845459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,2560,0.18562560081481932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,2048,0.19857385953267415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,12288,0.7763711929321289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,2048,0.15725013415018718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,2048,0.17851519584655762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,1536,0.1534293333689372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,1536,0.13565866152445477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,1536,0.1330453316370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,1024,0.11910293102264405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,1024,0.09540159702301025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,768,0.08664000034332275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,768,0.07979626655578613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,8192,0.5305578549702962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,512,0.07161706288655599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,512,0.0705013354619344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,5120,0.3342624028523763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,256,0.04369279940923055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,256,0.051235198974609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,3584,0.24558080037434898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,128,0.03484479983647664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,3072,0.21422826449076332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,128,0.05961706638336182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,64,0.04643733501434326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,64,0.06007359822591146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,8192,32,0.046913067499796554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,8192,32,0.059114666779836025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,2560,0.18787946701049804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,1024,0.10063146750132243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,768,0.0879701296488444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,512,0.07353066603342692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,256,0.06103786627451578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,8192,128,0.05574719905853272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,16384,1.3293504079182943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,65536,3.786761474609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,16384,0.9462677637736002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,65536,3.917868804931641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,51200,2.988042704264323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,12288,0.7316544214884441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,12288,1.02347412109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,12288,0.8039168039957681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,51200,3.141588338216146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,10240,0.880846913655599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,51200,4.309968058268229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,65536,5.512267557779948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,10240,0.6143797556559245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,8192,0.5177632013956706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,8192,0.7414154688517253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,7168,0.43932479222615556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,7168,0.624179204305013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,7168,0.433572260538737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,6144,0.412335999806722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,6144,0.5303711891174316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,6144,0.3780213356018066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,5120,0.31362667083740237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,5120,0.44602133433024094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,4096,0.251038932800293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,4096,0.2677258809407552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,4096,0.36412054697672525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,3584,0.31682453155517576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,3584,0.23272746404012046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,3072,0.2784576098124186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,3072,0.21399680773417154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,8192,0.513211727142334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,2560,0.17687360445658368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,2560,0.23416426976521812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,2048,0.1743029276529948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,2048,0.1462442715962728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,16384,0.948570696512858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,1536,0.1374165376027425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,1536,0.1266709327697754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,1536,0.1239466667175293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,1024,0.10765546957651775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,1024,0.08736000061035157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,1024,0.11530133088429768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,768,0.07437439759572348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,768,0.07694613138834636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,512,0.05912853479385376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,512,0.06133759816487631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,10240,0.5939306894938151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,256,0.03844053347905477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,256,0.047678931554158525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,256,0.060736000537872314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,128,0.030356266101201373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,128,0.04285866816838582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,5120,0.30349547068277993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,64,0.028509867191314698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,3584,0.22367253303527831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,64,0.0424511988957723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,7168,32,0.028883200883865357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,7168,32,0.04237333138783773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,3072,0.19477866490681967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,2560,0.16734933853149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,768,0.08112533092498779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,512,0.06605973243713378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,2048,0.14362986882527667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,7168,128,0.05160426696141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,65536,3.5272895812988283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,16384,1.2319967905680338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,51200,2.650795745849609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,16384,0.8453664143880208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,16384,0.8864021301269531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,12288,0.9338997522989908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,65536,4.85473378499349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,12288,0.6594805399576823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,51200,2.7909365336100262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,51200,3.9129908243815104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,12288,0.7030656178792317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,10240,0.5655477523803711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,10240,0.7867594401041667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,8192,0.5977322896321614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,8192,0.46303361256917314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,7168,0.4009045282999675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,7168,0.5575989405314128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,6144,0.35305706659952796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,6144,0.4619658788045247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,6144,0.329805850982666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,5120,0.29445012410481775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,5120,0.3988586743672689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,4096,0.31504427591959633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,4096,0.23677120208740235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,3584,0.2080394744873047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,3584,0.27536532084147136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,3584,0.19416747093200684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,3072,0.24137813250223794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,3072,0.1806154727935791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,2560,0.18664320309956867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,65536,3.5878580729166663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,2560,0.15599039395650227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,2048,0.15246507326761882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,2560,0.1845919926961263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,2048,0.1307029326756795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,2048,0.15135040283203124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,1536,0.1179317315419515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,1536,0.11029760042826335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,1024,0.08840320110321045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,8192,0.40520747502644855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,10240,0.5196511904398601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,1024,0.08171626726786295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,768,0.06884586811065674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,768,0.06851627031962076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,512,0.05111680030822754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,512,0.05581226746241251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,7168,0.3690901438395182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,256,0.03481280008951823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,256,0.04627626736958822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,256,0.05383466482162476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,5120,0.26307733853658044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,128,0.028123732407887774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,128,0.039323735237121585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,64,0.023782400290171306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,64,0.03924479881922404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,4096,0.21701760292053224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,6144,32,0.024566400051116943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,6144,32,0.03922986586888631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,3072,0.16942933400472004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,1536,0.10237332979838054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,1024,0.08039360046386719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,768,0.06991253693898519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,512,0.058422398567199704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,6144,128,0.04452693462371826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,65536,3.011798350016276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,16384,1.0067071914672852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,16384,0.7681578954060873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,16384,0.7376415888468425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,65536,4.069780222574869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,12288,0.8011423746744791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,51200,2.3343882242838543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,12288,0.5935093561808269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,10240,0.6420138676961262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,10240,0.4602698644002278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,8192,0.4956170717875163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,10240,0.5307392120361328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,51200,3.194419097900391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,8192,0.41426560084025066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,7168,0.3573205312093099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,7168,0.4758303960164388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,6144,0.37575785319010413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,6144,0.3052053451538086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,6144,0.3279648145039876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,5120,0.31803948084513345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,5120,0.2590250651041667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,4096,0.2533386707305908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,4096,0.21069119771321615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,4096,0.2045226732889811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,3584,0.21728960673014322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,3584,0.18952107429504395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,3584,0.1755392074584961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,3072,0.18459307352701823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,3072,0.1651306629180908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,2560,0.15809173583984376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,65536,3.1143211364746093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,2560,0.1450655937194824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,2560,0.139301331837972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,2048,0.12917546431223553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,12288,0.5260341326395671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,51200,2.3866859436035157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,2048,0.1181930700937907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,2048,0.12391680081685383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,1536,0.12291626930236817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,1536,0.09162453015645346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,1536,0.10307626724243164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,1024,0.08419199784596762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,1024,0.06952533721923829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,768,0.06019946734110514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,768,0.058514134089152015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,512,0.040106666088104245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,512,0.04745493332544963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,8192,0.35551573435465494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,256,0.027080533901850383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,256,0.03869866530100505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,7168,0.3159690539042155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,128,0.02300800085067749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,128,0.035796264807383224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,5120,0.23185812632242836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,64,0.02146773338317871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,64,0.03621546824773152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,5120,32,0.022587732474009196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,5120,32,0.03705919981002807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,3072,0.1492842674255371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,1024,0.07038186391194662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,768,0.062326399485270176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,512,0.05128959814707438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,256,0.042733867963155106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,5120,128,0.03922026554743449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,16384,0.7646432240804036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,65536,2.598066202799479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,16384,0.6592277526855469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,65536,3.3850868225097654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,16384,0.6492511749267578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,12288,0.5821674982706706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,51200,2.039141337076823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,51200,1.9786421457926433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,10240,0.5224234580993652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,10240,0.43808533350626633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,51200,2.6356821695963544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,8192,0.43419198989868163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,8192,0.30739094416300455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,8192,0.3646922747294108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,7168,0.36066560745239257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,7168,0.33412799835205076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,6144,0.2842506726582845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,12288,0.5052703857421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,5120,0.2601173400878906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,5120,0.22243626912434897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,4096,0.1972394625345866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,4096,0.189957332611084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,6144,0.2933333396911621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,3584,0.17147839864095052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,3584,0.16052160263061524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,12288,0.4537055969238281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,3072,0.15449919700622558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,3072,0.14745492935180665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,3072,0.133462397257487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,65536,2.573552958170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,10240,0.3784106572469076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,2560,0.1291317303975423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,2560,0.10889386336008708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,2048,0.10518933137257894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,2560,0.13960426648457844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,2048,0.10041173299153645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,1536,0.08494933446248373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,7168,0.26438934008280435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,1536,0.08132373491923015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,1024,0.05610773166020712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,1024,0.0625983993212382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,768,0.04781653483708699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,6144,0.23103466033935546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,768,0.054720000425974524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,512,0.03561066786448161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,5120,0.19454186757405598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,512,0.042878933747609455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,256,0.024510933955510458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,256,0.036635732650756835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,4096,0.16085227330525714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,128,0.018786134322484334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,128,0.03217386603355408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,128,0.032706133524576825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,1024,0.062033065160115565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,64,0.01677653392155965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,4096,32,0.017663999398549398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,64,0.03170560002326965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,32,0.03197759985923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,3584,0.1423829396565755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,4096,512,0.04568106730779012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,2048,0.09464213053385416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,1536,0.07564160029093424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,768,0.051344001293182374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,4096,256,0.03527359962463379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,51200,2.335069783528646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,16384,0.6874922434488933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,65536,2.4549524943033854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,65536,2.983887990315755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,12288,0.5578730901082356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,16384,0.6045802434285481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,51200,1.9306624094645184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,12288,0.47873814900716144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,10240,0.42392746607462567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,10240,0.402510929107666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,8192,0.3510080019632975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,8192,0.32351786295572915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,7168,0.3015242576599121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,7168,0.286683718363444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,65536,2.6338422139485678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,6144,0.2733109474182129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,16384,0.5789951960245768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,6144,0.23136746088663734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,6144,0.2567082722981771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,12288,0.4448383967081706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,5120,0.2041269302368164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,51200,2.028298695882161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,4096,0.17381547292073568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,4096,0.17066879272460939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,3584,0.1534527937571208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,10240,0.39845120112101234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,3584,0.1503946622212728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,3072,0.13193493684132893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,3072,0.13100799719492595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,8192,0.30015252431233724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,2560,0.11084160010019939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,2560,0.11292586326599122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,2048,0.09004800319671631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,2048,0.093995730082194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,7168,0.26821012496948243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,5120,0.21125440597534179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,1536,0.07100266615549723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,1536,0.07742613156636556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,1536,0.0671989361445109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,1024,0.0509173313776652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,1024,0.06020373503367106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,768,0.04185493389765422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,768,0.05103679895401001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,5120,0.1922005335489909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,512,0.03150186737378438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,512,0.04222186803817749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,4096,0.15426239967346192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,256,0.02181866765022278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,3584,0.14064745903015136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,256,0.03370240132013957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,3072,0.11907839775085449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,128,0.029798400402069092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,2560,0.10441706975301106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,64,0.014562132954597472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,64,0.029783467451731365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,32,0.01513920029004415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3584,32,0.029735465844472248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,2048,0.08673173586527506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,1024,0.052705065409342444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,768,0.04267946481704712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3584,128,0.016452266772588094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,512,0.03389120101928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,65536,2.1826431274414064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,65536,2.533233133951823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,256,0.027956267197926838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,51200,2.010842641194661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3584,128,0.022873600323994957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,16384,0.5607413609822591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,16384,0.5766293207804363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,12288,0.49783573150634763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,12288,0.4238752047220866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,10240,0.35474452972412107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,10240,0.38253119786580403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,8192,0.30220053990681967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,8192,0.3059605280558268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,51200,1.6959157307942707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,7168,0.25620479583740235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,7168,0.2566837310791016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,6144,0.21522132555643717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,6144,0.22340799967447916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,65536,2.3401097615559894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,5120,0.20363094011942545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,16384,0.5466559727986653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,5120,0.20797333717346192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,12288,0.41225385665893555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,5120,0.18634559313456217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,4096,0.15621973673502604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,4096,0.1650602658589681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,3584,0.13205653031667072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,10240,0.35647360483805335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,3584,0.14089706738789876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,3072,0.11318613688151043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,51200,1.813523228963216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,3072,0.12310293515523274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,2560,0.10681813557942707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,2560,0.10551893711090088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,8192,0.28352320988972984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,2048,0.07899946371714274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,2048,0.08845333258310953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,1536,0.06113066673278809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,1536,0.07262506484985351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,7168,0.24933973948160806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,1024,0.04432106812795003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,6144,0.20769920349121093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,1024,0.05501546859741211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,4096,0.14585280418395996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,768,0.03568533261617025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,768,0.046667734781901046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,3584,0.1283029317855835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,512,0.027852799495061236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,3072,0.11005120277404785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,512,0.03965013424555461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,256,0.0197653333346049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,256,0.03195306658744812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,2560,0.09677866299947104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,128,0.015107199549674988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,2048,0.07928746541341146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,128,0.027048534154891966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,64,0.013224533200263977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,64,0.027804799874623615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,3072,32,0.013585066795349121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,1536,0.06064000129699707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,3072,32,0.027088000377019243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,1024,0.04844053188959758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,65536,2.091537094116211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,65536,1.9738783518473308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,768,0.03824426730473836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,512,0.030631466706593828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,51200,1.6523146311442058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,256,0.025006934007008867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,3072,128,0.02063360015551249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,16384,0.46956478754679365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,16384,0.5042421340942382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,12288,0.35203094482421876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,12288,0.3835359891255697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,10240,0.29580373764038087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,10240,0.32176319758097327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,51200,1.563300323486328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,8192,0.23862187067667642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,7168,0.22259200414021812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,8192,0.2620927969614665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,7168,0.25640320777893066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,6144,0.19595306714375813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,6144,0.23212480545043945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,12288,0.3532341321309408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,5120,0.15046186447143556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,5120,0.1833343982696533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,65536,1.9679968516031903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,4096,0.1421237309773763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,16384,0.4601578712463379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,6144,0.17530666987101237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,4096,0.17167253494262696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,3584,0.10713600317637126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,10240,0.3000255902608236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,3584,0.14740907351175944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,3072,0.09378240108489991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,3072,0.12165333429972332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,2560,0.08019306659698486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,2560,0.10192853609720867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,7168,0.20478933652242026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,2048,0.06389973163604737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,51200,1.5052096048990884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,2048,0.08245013554890951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,1536,0.05379840135574341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,1536,0.0670687993367513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,8192,0.2333738644917806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,1024,0.0392576018969218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,1024,0.05072106520334879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,3072,0.09062826633453369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,768,0.03191999991734822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,768,0.043459200859069826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,1536,0.05058026711146037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,512,0.024877866109212242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,512,0.03613226811091105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,5120,0.14763733545939128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,4096,0.12259840170542399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,256,0.017668267091115318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,2560,0.08180373509724935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,128,0.013637333114941915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,128,0.026395734151204425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,3584,0.10653013388315838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,64,0.011575466394424439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,64,0.02645439902941386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2560,32,0.012054399649302164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,32,0.02560960054397583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,1024,0.04221546649932861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,2048,0.06562773386637369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,65536,1.6125675201416017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2560,256,0.02993280092875163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,65536,1.7917504628499348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,51200,1.2297728220621744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,16384,0.41101118723551433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,51200,1.3993792215983072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,16384,0.4818880081176758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,12288,0.2827199935913086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,128,0.01644053359826406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,12288,0.36704320907592775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,10240,0.23720000584920248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,10240,0.3041621208190918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,512,0.025337600708007814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,768,0.031522132953008014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,8192,0.18761812845865886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,8192,0.2385632038116455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2560,256,0.01943146586418152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,7168,0.1678101380666097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,7168,0.21198293368021645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,6144,0.14344107309977214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,6144,0.18243306477864582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,65536,1.7109643300374349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,5120,0.12407893339792889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,5120,0.16067946751912435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,12288,0.3116853396097819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,4096,0.09964906374613444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,4096,0.1302346706390381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,16384,0.41316585540771483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,3584,0.08623147010803223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,3584,0.11237226327260334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,51200,1.3423018137613931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,3072,0.08126400311787924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,3072,0.10839893023173015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,10240,0.2640170733133952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,2560,0.06368639866511026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,2560,0.08460693359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,7168,0.1864938735961914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,2048,0.05176213184992472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,8192,0.21018452644348146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,2048,0.07086506684621176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,1536,0.040161065260569256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,1536,0.05695679982503256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,6144,0.1568288008371989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,1024,0.028484266996383668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,1024,0.04225493272145589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,4096,0.11168106396993001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,768,0.023898667097091673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,3072,0.08181653022766114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,768,0.03615359862645467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,512,0.018092799186706542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,512,0.029883732398351032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,5120,0.13354560534159343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,256,0.012471466263135274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,256,0.024293333292007446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,1536,0.04508906602859497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,2560,0.07215039730072022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,128,0.009703466296195984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,128,0.02450773318608602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,64,0.01030613382657369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,64,0.02434879938761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,2048,32,0.01086186667283376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,2048,32,0.02440320054690043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,3584,0.0960863987604777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,65536,1.2661056518554688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,65536,1.638859685262044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,2048,0.058297598361968996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,1024,0.03709119955698649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,51200,0.9958293279012045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,51200,1.287607447306315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,16384,0.3271797180175781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,16384,0.4204629262288411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,12288,0.22738134066263832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,12288,0.3294879913330078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,128,0.014476799964904785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,512,0.022190932432810465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,10240,0.1922005335489909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,10240,0.2721205393473307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,10240,0.2247231960296631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,8192,0.1513802687327067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,8192,0.2198560078938802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,8192,0.170853328704834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,768,0.028219733635584516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,7168,0.1315872033437093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,6144,0.1137781302134196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,7168,0.19591360092163085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,2048,256,0.016807466745376587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,65536,1.3921749114990234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,5120,0.09834880034128825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,6144,0.1694111982981364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,4096,0.08015039761861166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,5120,0.1522165298461914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,4096,0.11860160032908122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,3584,0.07004799842834472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,51200,1.0909994761149089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,16384,0.3364842732747396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,3072,0.061501868565877284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,3072,0.0920202652613322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,12288,0.2667178789774577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,2560,0.050385065873463954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,3584,0.10441066424051922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,2560,0.07835946877797445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,2560,0.05993280013402304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,2048,0.04115946690241496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,2048,0.06403199831644693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,1536,0.03238079945246379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,1536,0.052067200342814125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,7168,0.15560320218404133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,1024,0.023082667589187623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,6144,0.13867200215657552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,1024,0.037962667147318524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,768,0.01858453353246053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,5120,0.11087360382080078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,768,0.03282986680666606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,512,0.015826132893562318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,768,0.023468800385793052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,4096,0.09135253429412842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,512,0.028092799584070842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,256,0.011076266566912334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,256,0.022986666361490885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,3584,0.07822186946868896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,128,0.00843946635723114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,128,0.021192532777786256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,3072,0.06859626770019531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,64,0.0074986666440963745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,64,0.020820266008377074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1536,32,0.008020266890525818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1536,32,0.020832000176111858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,2048,0.04916373491287231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,1536,0.03880319992701213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,65536,0.8536853154500326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,1024,0.030009599526723225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,512,0.01895466645558675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,65536,1.4439807891845704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,256,0.015064533551534018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,16384,0.20662506421407065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,16384,0.37449280420939124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,12288,0.1564469337463379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1536,128,0.013175466656684875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,51200,1.1402047475179038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,12288,0.2834570566813151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,51200,0.6922719955444336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,10240,0.12966293493906658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,8192,0.10444800059000652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,10240,0.23989653587341309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,8192,0.19323093096415203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,7168,0.0933141311009725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,7168,0.17069226900736492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,6144,0.07914026578267416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,6144,0.14891519546508789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,65536,1.2018730163574218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,5120,0.06788053512573242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,16384,0.3007722536722819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,5120,0.1313045342763265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,4096,0.054331731796264646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,51200,0.9437760035196939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,4096,0.10539093017578124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,12288,0.22722133000691733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,3584,0.047695998350779215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,3584,0.0921727975209554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,3072,0.04123626550038655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,10240,0.19134613672892253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,3072,0.0800277312596639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,2560,0.03464639981587728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,2560,0.0668405294418335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,8192,0.15702932675679523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,7168,0.13722559611002605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,2048,0.05411413510640463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,6144,0.11808853149414063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,1536,0.022055466969807945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,1536,0.04039573272069295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,2048,0.028231465816497804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,1024,0.01576959987481435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,1024,0.03088639974594116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,5120,0.098362668355306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,768,0.012647466858228049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,4096,0.08071040312449138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,768,0.027003733317057292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,512,0.009593600034713745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,3584,0.07085546652475992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,512,0.02304960091908773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,256,0.006826666494210561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,256,0.02105813423792521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,3072,0.061052799224853516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,128,0.006196266909440359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,128,0.018551466862360637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,2560,0.05242026646931967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,64,0.005701333284378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,64,0.01893333395322164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,1024,32,0.006298666695753734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,1024,32,0.018513067563374837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,2048,0.04256213506062825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,1536,0.03284800052642822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,65536,0.6268213272094727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,51200,0.5243829409281413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,65536,1.352838389078776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,1024,0.024371200799942018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,16384,0.15627412796020507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,768,0.01992320020993551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,51200,1.0771648406982421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,512,0.015871999661127727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,16384,0.3490847905476888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,12288,0.12140693664550781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,256,0.01229759951432546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,10240,0.0999573310216268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,12288,0.26699625651041664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,1024,128,0.010866133371988933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,10240,0.22470720609029135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,8192,0.08053759733835855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,7168,0.07092373371124268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,8192,0.1830645402272542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,7168,0.16277653376261395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,6144,0.061516801516215004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,6144,0.14213120142618815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,65536,0.9552842458089194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,5120,0.054604800542195644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,5120,0.12423466841379802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,51200,0.7342538833618164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,16384,0.24211840629577636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,4096,0.04228586753209432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,4096,0.09997333685557047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,12288,0.18362453778584797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,3584,0.037800534566243486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,10240,0.1573813279469808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,3584,0.08805867036183676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,8192,0.1263914664586385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,3072,0.033787735303243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,3072,0.07583146890004476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,7168,0.11033493677775066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,2560,0.028413865963617963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,2560,0.06209386587142944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,2048,0.023703465859095253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,2048,0.05008960167566935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,6144,0.09577066898345947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,1536,0.019181867440541588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,1536,0.03856213490168254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,1024,0.014252799749374389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,5120,0.08019839922587077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,1024,0.029597866535186767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,768,0.011860266327857971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,1024,0.020937599738438926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,768,0.02691520055135091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,512,0.009610666831334432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,4096,0.0649397333463033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,512,0.023405865828196207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,256,0.006285866598288219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,256,0.019687465826670327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,256,0.01035520037015279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,3584,0.05707199970881144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,128,0.005606399973233541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,128,0.018191999197006224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,64,0.005097599824269613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,64,0.0180565337340037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,768,32,0.005231999854246775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,3072,0.050327467918396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,2560,0.04284160137176514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,768,32,0.01772159934043884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,2048,0.035282135009765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,65536,0.4018783887227376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,1536,0.027585067351659137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,51200,0.3109930674235026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,16384,0.10211199919382732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,16384,0.3293397267659505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,65536,1.2411253611246744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,51200,0.7414037068684896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,12288,0.07913920084635416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,51200,0.9744768142700195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,10240,0.06593279838562012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,12288,0.2440938631693522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,10240,0.2073823928833008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,8192,0.05321066776911417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,8192,0.16989760398864745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,768,0.01716266671816508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,7168,0.04793813228607178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,512,0.013507200280825296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,7168,0.1509973367055257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,6144,0.041568001111348465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,768,128,0.008814932902654012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,6144,0.13137919902801515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,5120,0.035323735078175864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,5120,0.11204373041788737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,5120,0.08110826810201009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,4096,0.028614399830500285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,4096,0.0912170648574829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,65536,0.9645439783732096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,3584,0.02532266577084859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,3584,0.08079466819763184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,16384,0.238808536529541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,3072,0.02193280061086019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,12288,0.18322666486104328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,3072,0.0678933302561442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,2560,0.019052799542744955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,2560,0.05491626660029093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,10240,0.153276793162028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,2048,0.01592639982700348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,2048,0.0435263991355896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,8192,0.12669226328531902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,1536,0.013009066383043924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,1536,0.033846398194630936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,7168,0.10900053183237712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,1024,0.00993280013402303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,1024,0.027088000377019243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,6144,0.09577813148498535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,768,0.008521599570910136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,768,0.025306665897369386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,4096,0.06477119922637939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,512,0.007125333448251088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,512,0.02257386644681295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,3584,0.0584768017133077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,256,0.0060703997810681665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,256,0.018931200106938682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,3072,0.04994026819864909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,128,0.005017599960168203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,128,0.017324799299240114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,2560,0.04249279896418254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,64,0.004440533121426901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,64,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,2048,0.0342410683631897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,512,32,0.0044714664419492085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,512,32,0.016718933979670204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,65536,0.25613226890563967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,1536,0.026614399751027425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,1024,0.019350399573644005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,51200,0.19189546902974447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,768,0.016772266228993735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,65536,1.160587692260742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,16384,0.06272000074386597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,512,0.012728533148765564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,51200,0.9138943990071615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,12288,0.04899306694666545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,16384,0.30295572280883787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,12288,0.23030400276184082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,10240,0.04113920132319133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,256,0.00941546658674876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,10240,0.19461867014567058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,8192,0.03315413395563761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,512,128,0.008046933511892954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,8192,0.15894187291463216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,7168,0.03024853269259135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,7168,0.14056000709533692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,6144,0.026858667532602947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,6144,0.12270613511403401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,65536,0.6467658360799153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,5120,0.023114667336146037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,51200,0.5127231915791829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,5120,0.10337066650390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,4096,0.01588586668173472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,4096,0.08402453263600668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,16384,0.17034346262613934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,3584,0.014177067081133523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,3584,0.07339626948038737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,12288,0.12888212998708087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,3072,0.012488533059755962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,3072,0.06137066682179769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,10240,0.1080672025680542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,2560,0.011169067025184632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,8192,0.0875157356262207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,2560,0.049352534612019855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,2048,0.009508267045021057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,2048,0.038812800248463945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,7168,0.07654293378194174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,1536,0.008195200065771738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,1536,0.030384000142415362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,6144,0.06641173362731934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,1024,0.006625066697597504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,1024,0.025565866629282636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,5120,0.056143999099731445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,768,0.00581226646900177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,768,0.022679466009140014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,4096,0.04609066645304362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,512,0.005411200225353241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,512,0.01984000007311503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,3584,0.041571199893951416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,256,0.0045621335506439206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,256,0.017935999234517417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,3072,0.03639359871546428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,128,0.0039818666875362395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,128,0.0166101336479187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,2560,0.031149866183598836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,64,0.003576533248027166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,64,0.016544000307718913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,256,32,0.0036874666810035707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,2048,0.024727465709050496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,256,32,0.016085333625475564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,1536,0.01972480018933614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,65536,0.22441493670145668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,51200,0.178654940923055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,1024,0.015067733327547708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,65536,1.1525877634684245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,16384,0.06415146589279175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,51200,0.9001162846883138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,768,0.012258133292198181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,16384,0.30086507797241213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,12288,0.04814186493555705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,12288,0.22927786509195963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,10240,0.04162773291269938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,512,0.010121599833170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,10240,0.19408639272054035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,256,0.007778133451938629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,8192,0.035336534182230636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,256,128,0.0067669332027435304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,8192,0.157861328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,7168,0.02916160027186076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,7168,0.14032425880432128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,6144,0.025986133019129436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,6144,0.12000319957733155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,65536,0.6376266479492188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,5120,0.02218773365020752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,5120,0.10183893044789631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,51200,0.49865067799886065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,4096,0.012668800354003907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,16384,0.16454826990763347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,4096,0.08203199704488119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,3584,0.012181333700815837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,3584,0.07087893486022949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,12288,0.12357652982076009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,3072,0.010016000270843506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,8192,0.08426453272501627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,10240,0.10464106400807698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,3072,0.05880320072174072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,3072,0.03523199955622355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,2560,0.008764800429344178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,2560,0.04633386532465617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,7168,0.07422506809234619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,2048,0.009026133020718892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,2048,0.036651734511057535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,1536,0.0065642664829889934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,1536,0.029692800839742024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,6144,0.06456106503804525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,1024,0.005467733244101206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,1024,0.024218666553497314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,5120,0.05332800149917603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,768,0.005126399795214335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,768,0.021713066101074218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,4096,0.044606932004292804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,512,0.0046069333950678505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,512,0.019761067628860474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,3584,0.03934400081634522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,256,0.00373333344856898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,256,0.016795732577641807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,128,0.003390933324893316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,128,0.016291200121243795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,2560,0.02863146662712097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,64,0.003201066702604294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,64,0.01569386621316274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,128,32,0.003253333270549774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,2048,0.02374826669692993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,128,32,0.015638400117556253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,1536,0.018503467241923012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,51200,0.17308160463968914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,65536,0.2174837271372477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,1024,0.014079999923706055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,16384,0.06477333307266235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,12288,0.047434667746226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,16384,0.29895572662353515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,12288,0.2242058595021566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,10240,0.03916159868240356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,8192,0.03360213438669841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,10240,0.18906985918680827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,65536,1.149132792154948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,51200,0.8984095891316732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,8192,0.15440533955891927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,7168,0.025464532772699992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,6144,0.022820266087849934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,768,0.01141866644223531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,6144,0.11904853185017902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,7168,0.1367797374725342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,5120,0.10077760219573975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,4096,0.010609066486358643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,3584,0.009443199634552002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,3072,0.00851093331972758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,4096,0.08094186782836914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,3584,0.06942186355590821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,2560,0.007785599927107494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,2048,0.007196799914042156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,3072,0.057601066430409756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,2560,0.045664000511169436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,1536,0.0059114664793014525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,2048,0.035179734230041504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,1536,0.02840320070584615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,768,0.004289066791534424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,1024,0.023925334215164185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,512,0.003868799904982249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,768,0.0213045338789622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,256,0.0033962666988372804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,512,0.01916586756706238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,128,0.0031413334111372627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,256,0.01774079998334249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,64,0.003044266750415166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,128,0.016116266449292503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,32,0.0031167998909950255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,64,0.015586133797963461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,512,0.009225599964459737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,64,32,0.015017599860827128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,256,0.007035733262697856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,65536,0.21615999539693198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,51200,0.17094933191935222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,16384,0.05630079905192057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,12288,0.04465173482894898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,10240,0.03719040155410767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4352,128,128,0.005931733548641205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,8192,0.03249920010566711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,12288,0.22343467076619467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,16384,0.29570560455322265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,7168,0.024911999702453613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,10240,0.18963947296142578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,5120,0.019373865922292073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,8192,0.15366400082906087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,5120,0.018978132804234823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,7168,0.13553919792175292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,6144,0.11804479757944744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,5120,0.09915200074513754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,3584,0.008770133058230083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,3072,0.007914666831493378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,3584,0.06859839757283528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,3072,0.05694080193837484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,2560,0.007097599903742473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,64,1024,0.004771199822425842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,2560,0.04503573179244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,65536,1.1432362874348958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,2048,0.006263466676076253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,1536,0.005390933156013489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,2048,0.03518720070521037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,1024,0.004389333228270212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,768,0.003970133264859518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,1024,0.025113600492477416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,512,0.003568000098069509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,768,0.021331199010213218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,256,0.003159466634194056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,512,0.01914773384730021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,128,0.002980266759792964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,256,0.016976000865300496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,64,0.0028490667541821797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,128,0.015621333320935567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,64,0.015446399648984274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,32,0.0150026669104894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,6144,0.022270933787027995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,4096,0.009816533327102661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,51200,0.894109853108724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,4096,0.0797706683476766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4352,32,1536,0.028531199693679808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4352,32,32,0.0027744000156720476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,16384,6.272899373372396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,12288,4.791768391927084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,12288,8.772965494791666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,16384,11.779862467447916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,16384,8.714959716796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,12288,7.163014221191406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,10240,4.6430816650390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,10240,7.431061299641927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,8192,3.4836926778157555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,8192,4.198592122395834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,8192,7.19429423014323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,7168,2.822130076090495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,10240,5.365692647298177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,7168,5.565495300292969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,51200,21.260221354166667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,7168,3.672723134358724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,6144,2.4670453389485676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,5120,2.066416041056315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,6144,4.672515360514323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,5120,3.751036834716797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,4096,1.698525873819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,5120,2.551678975423177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,4096,3.0441024780273436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,3584,1.4840843200683593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,6144,3.1126859029134115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,3584,2.6420939127604166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,3584,1.83910395304362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,3072,1.2849653879801433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,3072,2.633778127034505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,2560,1.1542485555013022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,2560,1.974106725056966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,2560,1.4847498575846354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,2048,0.9958688100179037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,4096,2.2068692525227864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,2048,1.7196917215983074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,1536,0.7117354710896809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,51200,30.953350830078126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,1536,0.9683317184448242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,1536,1.2075509389241537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,1024,0.5270410537719726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,1024,0.8580458958943685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,768,0.43460693359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,768,0.6060618718465169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,1024,0.6966016133626302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,512,0.3074847857157389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,512,0.47753705978393557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,768,0.6470368067423503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,3072,1.5299636840820312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,256,0.254476801554362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,256,0.2722058614095052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,128,0.23825599352518717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,128,0.2366432030995687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,256,0.4448277473449707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,64,0.22128426233927406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,32,0.21595093409220376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,32,0.2223434607187907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,128,0.4307125409444173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,2048,1.064028803507487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,65536,64,0.228929074605306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,65536,512,0.47529067993164065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,65536,51200,38.697705078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,16384,5.607464599609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,16384,9.420149739583334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,16384,6.484835306803386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,51200,16.599980672200523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,12288,8.045884704589843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,65536,21.467466227213542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,12288,3.8614273071289062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,12288,5.223385620117187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,10240,3.394765981038412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,51200,26.601123046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,10240,5.926194254557291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,8192,2.8511456807454425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,8192,4.74258778889974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,10240,4.016814931233724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,51200,29.43666178385417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,7168,2.459833526611328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,7168,4.518635559082031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,8192,3.1717791239420574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,6144,2.0312522888183593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,6144,2.7327646891276043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,6144,3.5736244201660154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,7168,2.8048095703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,5120,1.6825205485026042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,4096,1.2754464467366537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,5120,2.875762176513672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,4096,1.5994132995605468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,4096,2.475823974609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,3584,1.139460245768229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,3584,2.144629414876302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,3584,1.3824298858642579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,3072,1.0364042917887368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,3072,1.1940799713134767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,3072,1.7944180806477863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,2560,0.8963551839192709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,2560,1.5231210072835286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,65536,35.09015706380208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,5120,1.9917835235595702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,65536,38.47458089192708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,2560,1.1158196767171225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,2048,0.6721397399902344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,1024,0.41205441157023115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,2048,1.1164714813232421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,1536,0.5167562802632649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,1536,0.9520437240600585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,1024,0.6416106541951497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,768,0.336896006266276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,768,0.49854081471761064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,512,0.3760117212931315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,512,0.26590399742126464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,768,0.4951264063517253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,256,0.20597012837727866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,128,0.18960426648457845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,128,0.18927680651346843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,256,0.314572811126709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,64,0.17495999336242676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,64,0.18370879491170247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,32,0.1708170731862386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,51200,32,0.17832214037577312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,51200,256,0.22603093783060707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,2048,0.8388458887736002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,1536,0.6872554779052734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,1024,0.5077664057413738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,512,0.37736854553222654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,51200,128,0.2918837229410807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,16384,1.7775637308756511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,16384,2.984996287027995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,51200,5.351815287272135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,65536,7.142816162109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,12288,2.2070549011230467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,12288,1.3310826619466147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,51200,7.515561421712239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,12288,1.5226688385009766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,10240,1.8450602213541667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,10240,1.1140628814697267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,16384,2.073964818318685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,8192,0.8797578811645508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,51200,9.110546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,8192,1.5174517313639322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,8192,1.0652725219726562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,7168,0.7585930506388346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,7168,1.3578623453776042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,6144,1.2082602183024087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,65536,12.15613505045573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,5120,0.919702402750651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,5120,0.631213887532552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,5120,0.6222378412882488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,4096,0.5239232063293457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,4096,0.7441386540730794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,6144,0.7057429631551106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,10240,1.265509287516276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,65536,10.05634053548177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,4096,0.5696426391601562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,3584,0.4094090779622396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,3584,0.44097067515055344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,3584,0.751804796854655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,3072,0.6250879923502605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,3072,0.35827414194742835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,2560,0.306989860534668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,2560,0.39463466008504233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,2048,0.25366293589274086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,2048,0.4081802686055501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,1536,0.20311466852823892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,7168,0.8656021118164062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,1536,0.30930452346801757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,1536,0.25912426312764486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,1024,0.196725336710612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,768,0.15616960525512696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,1024,0.16735572814941407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,768,0.12419733206431072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,512,0.11957653363545735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,6144,0.738650639851888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,768,0.16831787427266437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,512,0.09291413625081381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,256,0.07795413335164389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,256,0.07698773543039958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,128,0.06147520144780477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,256,0.109443203608195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,128,0.07079786459604899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,2560,0.45671892166137695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,64,0.05586346785227457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,16384,32,0.05619839827219645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,64,0.06987093289693197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,16384,32,0.07000426451365152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,3072,0.3847807884216309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,2048,0.2789525349934896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,1024,0.17411093711853026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,512,0.12722667058308917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,16384,128,0.09998079935709635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,16384,1.3434304555257162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,16384,2.3719520568847656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,65536,5.444454447428385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,16384,1.5815040588378906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,12288,1.005305608113607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,51200,4.207957458496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,12288,1.678280512491862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,51200,5.1682378133138025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,10240,0.8509141286214194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,10240,0.9756522496541342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,10240,1.4787637074788411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,65536,7.4463348388671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,8192,0.6903722763061524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,65536,8.78590087890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,51200,6.899588012695313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,8192,1.089959462483724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,7168,0.9973834355672201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,7168,0.6058272043863933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,7168,0.6563103993733723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,6144,0.5137333234151205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,6144,0.7962890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,5120,0.44034773508707686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,5120,0.7755168279012044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,4096,0.36330347061157225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,4096,0.5240288098653157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,12288,1.1224180857340493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,3584,0.31604480743408203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,3584,0.5259946823120117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,3072,0.2716010729471842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,3072,0.46394879023234054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,3072,0.3420053482055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,2560,0.331989320119222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,2560,0.26315733591715496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,2560,0.2953023910522461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,2048,0.28569707870483396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,2048,0.23830933570861818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,1536,0.2259893258412679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,1536,0.1581120014190674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,1024,0.14885120391845702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,8192,0.7679733276367188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,1024,0.1195029338200887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,768,0.11546453634897869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,768,0.09850666522979737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,6144,0.5517941157023112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,512,0.09176533222198487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,2048,0.2115605354309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,512,0.0841439962387085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,256,0.06164906819661459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,5120,0.4762463887532552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,256,0.06623786687850952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,128,0.049711998303731283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,256,0.08814506530761719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,128,0.059087999661763514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,64,0.04288640022277832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,4096,0.382478936513265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,12288,32,0.043722665309906004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,64,0.05817279815673828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,3584,0.3388490676879883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,12288,32,0.05998293161392212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,1536,0.1751786708831787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,1024,0.13538133303324382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,768,0.1196885347366333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,512,0.099562668800354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,12288,128,0.07640960216522216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,16384,1.1332288106282553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,16384,1.8485333760579425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,65536,4.621001688639323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,51200,3.7725802103678383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,12288,0.8589994430541992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,12288,1.363763173421224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,51200,4.429770914713542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,12288,0.970093854268392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,10240,1.1965994517008463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,51200,5.598665364583334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,65536,7.180788167317708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,10240,0.8264938354492187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,8192,1.0042549133300782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,8192,0.6051189422607421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,8192,0.6523573557535808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,7168,0.793605359395345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,10240,0.7294581095377605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,7168,0.5565557479858398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,6144,0.6978485107421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,6144,0.45840638478597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,5120,0.38433494567871096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,6144,0.4900757471720378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,5120,0.5260479927062989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,16384,1.2913642883300782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,4096,0.46564159393310545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,4096,0.3063562711079916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,3584,0.29942614237467446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,65536,6.158209228515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,3584,0.42586666742960616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,3072,0.32948694229125974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,3072,0.278767999013265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,3072,0.2639839967091878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,2560,0.29469226201375326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,2560,0.21207893689473473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,2048,0.17764479319254559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,2048,0.24212907155354818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,5120,0.42342933019002277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,1536,0.186627197265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,1536,0.1318666696548462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,1024,0.11991893450419108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,3584,0.28868373235066735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,1024,0.09859093030293783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,768,0.09524160226186117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,768,0.08295040130615235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,7168,0.5529685338338216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,512,0.07751999696095785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,512,0.07630720138549804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,512,0.0979914665222168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,256,0.04866666793823242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,256,0.05703786611557007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,256,0.07250879605611166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,128,0.04056426684061686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,4096,0.31805121103922523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,64,0.036432000001271565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,64,0.05103466510772705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,10240,32,0.03699093262354533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,32,0.05132586558659872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,2560,0.21559359232584635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,2048,0.1827082633972168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,10240,128,0.05012586514155069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,1536,0.1474847952524821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,1024,0.11451413631439208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,768,0.10124800205230713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,10240,128,0.0635424017906189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,16384,1.5678496042887369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,16384,0.9925589243570963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,51200,3.1533673604329424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,65536,4.071178690592448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,51200,3.278247578938802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,12288,0.7991829554239909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,12288,0.7837781270345052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,10240,0.6327925364176432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,65536,5.9673100789388025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,10240,1.0118645350138347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,51200,4.8806304931640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,8192,0.5148074785868327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,8192,0.7573066711425781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,12288,1.1452330271402995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,7168,0.6794890721638998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,7168,0.4455733299255371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,6144,0.5427594502766927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,6144,0.4175104141235352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,6144,0.38706026077270506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,5120,0.3418570518493652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,16384,0.9868885040283203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,5120,0.4641258557637532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,5120,0.3309258778889974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,4096,0.30775467554728186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,4096,0.2927466710408529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,3584,0.31852906545003257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,3584,0.25102292696634926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,3584,0.2504981358846029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,3072,0.3169610659281413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,3072,0.21146772702534994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,3072,0.21597973505655924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,2560,0.2590933322906494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,65536,4.59776865641276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,2048,0.21071999867757163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,2048,0.1784181276957194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,2048,0.17024106979370118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,1536,0.1637941360473633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,10240,0.6334197362263997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,4096,0.3739232063293457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,1536,0.12377813657124836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,1024,0.1020095984141032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,1024,0.08960426648457845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,768,0.08075093428293864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,8192,0.49846719106038406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,768,0.07470293045043945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,512,0.06155733267466227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,512,0.05854933261871338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,256,0.03988693157831828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,256,0.04883413314819336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,2560,0.17768534024556476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,7168,0.4460032145182292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,128,0.033802668253580734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,128,0.043835731347401936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,64,0.03060479958852132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,64,0.043831467628479004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,8192,32,0.03084160089492798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,8192,32,0.04421226580937703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,2560,0.17819946606953938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,1536,0.11977279980977376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,1024,0.09498133659362792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,768,0.08347307046254476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,512,0.0694432020187378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,256,0.05750293334325155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,8192,128,0.05258133411407471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,16384,1.296133295694987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,65536,5.180296325683594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,16384,0.9285717646280924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,51200,2.805079396565755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,65536,3.542401123046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,12288,0.6829877217610677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,12288,0.6941706975301106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,12288,0.984440549214681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,10240,0.7846090952555339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,51200,4.08108164469401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,10240,0.5826453526814779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,8192,0.6578688303629557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,8192,0.46201492945353195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,8192,0.46338666280110674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,7168,0.5768853505452474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,7168,0.4059871991475423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,6144,0.49227412541707355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,6144,0.37545385360717776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,6144,0.3535999933878581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,5120,0.3100501378377279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,5120,0.413262939453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,4096,0.2581397374471029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,4096,0.32171414693196615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,3584,0.29149014155069985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,7168,0.42789653142293294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,3584,0.2282709280649821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,16384,0.8826101303100586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,3072,0.24723626772562662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,3072,0.20471253395080566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,65536,3.770972696940104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,2560,0.20766080220540367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,2560,0.17383893330891925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,10240,0.5672320048014323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,2048,0.16471360524495443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,2048,0.13972266515096027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,51200,3.0706891377766925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,1536,0.13193706671396893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,1536,0.10681386788686116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,1024,0.08982186317443848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,1024,0.08096853097279867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,5120,0.28952000935872396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,768,0.06994773546854655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,768,0.06763199965159097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,4096,0.2390378634134928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,512,0.05283519824345907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,512,0.054324265321095785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,3584,0.21160853703816734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,256,0.03505920171737671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,256,0.04505386749903361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,256,0.05192000071207682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,128,0.03028800090154012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,3072,0.18625493049621583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,128,0.03992960055669149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,128,0.04786239862442017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,64,0.026281599203745527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,7168,32,0.027078400055567425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,64,0.04096000194549561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,2560,0.15946346918741863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,7168,32,0.04032320181528727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,2048,0.1359242598215739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,1536,0.11063786347707112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,1024,0.0875221331914266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,768,0.07568639914194743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,7168,512,0.06236799955368042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,16384,1.102930196126302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,65536,3.143716176350911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,16384,0.7782400131225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,51200,2.410460917154948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,12288,0.6014549255371093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,51200,2.6639030456542967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,12288,0.5987360000610351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,12288,0.8007680257161459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,51200,3.545912424723307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,65536,4.511779276529948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,10240,0.5454730351765951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,10240,0.7027530670166016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,8192,0.42455040613810224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,8192,0.5702143987019856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,7168,0.393995730082194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,7168,0.47234026590983075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,7168,0.35204054514567057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,6144,0.4213706652323405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,6144,0.3334613482157389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,5120,0.32551466623942055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,5120,0.2772831916809082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,4096,0.22426986694335938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,4096,0.26852480570475257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,3584,0.24487573305765786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,3584,0.20736427307128907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,3584,0.19878400166829427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,65536,3.5256116231282553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,3072,0.21498452822367348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,16384,0.7775615692138672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,3072,0.17539946238199872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,3072,0.19231360753377277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,2560,0.1931381384531657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,2560,0.15702080726623535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,2048,0.13983786900838216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,2048,0.1200170675913493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,10240,0.4937589327494304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,2048,0.12467947006225585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,1536,0.11910933653513592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,1536,0.0952789306640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,1536,0.1146773338317871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,1024,0.07508479754130046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,1024,0.07641599973042806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,1024,0.07613226572672525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,8192,0.3868351936340332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,768,0.06315733194351196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,512,0.048104532559712726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,768,0.06167466640472412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,512,0.052266665299733485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,256,0.031674667199452715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,6144,0.3018453280131022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,256,0.04034239848454793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,128,0.02561066746711731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,256,0.04755520025889079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,128,0.03691413402557373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,128,0.04376320044199626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,64,0.022959999243418374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,6144,32,0.023297067483266196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,64,0.03731840054194133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,6144,32,0.03731306791305542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,4096,0.20827840169270834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,512,0.0548960010210673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,5120,0.25410985946655273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,2560,0.13866559664408368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,6144,768,0.06483199993769327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,65536,2.70570551554362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,16384,0.8344661076863608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,65536,2.8094207763671877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,16384,0.6837930679321289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,51200,2.123595682779948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,16384,0.6760309219360352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,65536,3.760539754231771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,51200,2.200177001953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,51200,2.9391146341959637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,12288,0.5456319808959961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,12288,0.7025472005208333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,12288,0.5068010648091634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,10240,0.46741228103637694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,10240,0.5620309193929036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,10240,0.4571882565816243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,8192,0.38691412607828773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,8192,0.48498986562093094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,7168,0.4133418718973796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,7168,0.33363625208536785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,7168,0.30683307647705077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,6144,0.35341653823852537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,6144,0.27777172724405924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,5120,0.29739306767781576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,5120,0.25190827051798503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,6144,0.29040425618489585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,4096,0.22147733370463052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,4096,0.1959328015645345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,3584,0.21075305938720704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,3584,0.1729856014251709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,3072,0.1714666684468587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,3072,0.15023040771484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,2560,0.14832960764567057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,3072,0.15109866460164387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,2560,0.13437333106994628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,2048,0.12153493563334147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,2048,0.10917332967122395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,1536,0.09086293379465739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,2048,0.11531519889831543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,1536,0.0853056033452352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,1024,0.06637973388036092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,1024,0.06552853186925253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,1024,0.07023893197377523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,768,0.04962559938430786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,768,0.05693120161692301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,768,0.05670933326085409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,512,0.039374931653340654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,512,0.04496106704076131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,256,0.02606826623280843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,256,0.03610346714655559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,4096,0.21132267316182457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,3584,0.1727242628733317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,128,0.0214464008808136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,128,0.033632000287373856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,64,0.019550933440526327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,64,0.033297065893809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,5120,32,0.020514132579167683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,5120,32,0.03399999936421712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,8192,0.3388789176940918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,5120,0.21887680689493813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,2560,0.12045653661092122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,1536,0.08492159843444824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,512,0.04728746811548869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,256,0.03959146738052368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,5120,128,0.03617386817932129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,65536,3.118921661376953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,65536,2.430572764078776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,65536,2.4471115112304687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,16384,0.6100309371948243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,51200,1.9389887491861981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,51200,2.3661343892415365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,12288,0.5795093536376953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,12288,0.4673151969909668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,10240,0.4692874590555827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,12288,0.4356117248535156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,16384,0.6964746475219726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,10240,0.4101397196451823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,8192,0.3730997403462728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,8192,0.2768469174702962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,7168,0.3095936139424642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,7168,0.29446080525716145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,7168,0.24804479281107583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,6144,0.2867424011230469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,6144,0.2141536076863607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,6144,0.2503328005472819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,5120,0.24456213315327963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,5120,0.18508159319559733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,5120,0.20538667043050132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,4096,0.19449493090311687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,4096,0.14958933194478352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,4096,0.16842880249023437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,3584,0.17177066802978516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,3584,0.13282240231831868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,3072,0.14843306541442872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,8192,0.33504425684611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,3072,0.12753173510233562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,3072,0.12989333470662434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,2560,0.11817920207977295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,2048,0.09658559958140055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,2560,0.11824426651000977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,2048,0.09518187046051026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,1536,0.07739946842193604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,1536,0.07452159722646078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,1024,0.053741868336995444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,1536,0.0698527971903483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,16384,0.5481088002522786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,1024,0.05951679944992065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,768,0.0460319995880127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,768,0.05047359863917032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,768,0.05192106564839681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,51200,1.8252128601074218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,512,0.033113600810368855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,256,0.022293333212534586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,512,0.04991360108057658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,256,0.03277013301849365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,256,0.04060266812642415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,128,0.01715946594874064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,3584,0.1494965394337972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,128,0.03184426625569661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,128,0.029634133974711103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,64,0.015301332871119181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,4096,32,0.01590933303038279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,64,0.031420799096425374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,10240,0.3524298667907715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,32,0.029498666524887085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,4096,512,0.04236799875895182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,2560,0.10072426795959473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,51200,2.08241818745931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,2048,0.08475733598073323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,65536,2.218763732910156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,4096,1024,0.05418773492177328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,65536,2.67698237101237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,16384,0.5620927810668945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,16384,0.6424704233805338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,51200,1.7171019236246745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,16384,0.5025930722554525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,12288,0.45902398427327473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,10240,0.4045066515604655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,12288,0.4460810661315918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,8192,0.33357973098754884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,8192,0.31144959131876626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,7168,0.2968981424967448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,7168,0.26123414039611814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,6144,0.2269418716430664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,6144,0.22511253356933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,10240,0.36039679845174155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,5120,0.1910666624704997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,65536,2.0601216634114583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,5120,0.18997546831766765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,4096,0.15519359906514485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,4096,0.15464746157328288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,4096,0.13749972979227704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,3584,0.13691627184549968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,3584,0.13713919321695964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,12288,0.37652158737182617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,10240,0.31646080017089845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,3072,0.12099626859029133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,3072,0.12257280349731445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,51200,1.7042655944824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,8192,0.25648852984110515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,2560,0.10185173352559407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,2560,0.09289279778798422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,2048,0.08392852942148844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,2560,0.12454079786936442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,1536,0.08086400032043457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,2048,0.08762239615122477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,7168,0.2285930633544922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,1536,0.06403626600901285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,1536,0.07499306996663412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,1024,0.04712213277816772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,1024,0.057479465007781984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,768,0.040746665000915526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,768,0.048266665140787764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,6144,0.19608426094055176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,512,0.029392000039418536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,512,0.039325865109761556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,2048,0.07860586643218995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,256,0.020550400018692017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,5120,0.16589760780334473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,256,0.03151359955469767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,128,0.01541973352432251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,128,0.02774080038070679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,3584,0.1226474682490031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,64,0.01328000028928121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,3072,0.10778773625691732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,64,0.027539199590682982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3584,32,0.013900799552599588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3584,32,0.0269322673479716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,1024,0.04969066778818766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,768,0.04389653205871582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,512,0.035665067036946614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,256,0.02925119996070862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3584,128,0.02659306724866231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,65536,2.3555625915527343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,65536,1.851210657755534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,51200,1.7779679616292317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,16384,0.5173653284708659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,16384,0.5199178695678711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,51200,1.634000015258789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,12288,0.4308170636494954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,12288,0.4290474573771159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,10240,0.33075520197550456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,10240,0.3528501192728678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,10240,0.31113173166910807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,8192,0.266428804397583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,8192,0.29475733439127605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,7168,0.24658667246500648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,7168,0.2546559969584147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,65536,2.0367242177327474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,7168,0.22132693926493324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,6144,0.21195947329203285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,6144,0.21581759452819824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,5120,0.167847474416097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,5120,0.19542293548583983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,5120,0.14978559811909992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,4096,0.1412874698638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,4096,0.1531221389770508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,3584,0.12088639736175537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,4096,0.12305920124053955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,3072,0.10306560198465983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,3584,0.13799999554951986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,3072,0.11427306334177653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,3072,0.10066773096720379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,2560,0.08743680318196614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,2560,0.09852480093638102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,2048,0.07286612987518311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,2048,0.08385706742604573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,2048,0.07005013624827067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,1536,0.0570741335550944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,1536,0.06675946712493896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,16384,0.43247038523356124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,51200,1.382593027750651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,1024,0.04078400135040283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,12288,0.33490667343139646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,1024,0.05975146691004435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,1024,0.043944533665974936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,768,0.03347520033518474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,512,0.02684906721115112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,512,0.03741333484649658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,512,0.03375786542892456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,256,0.018473599354426065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,256,0.0291978657245636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,8192,0.22884906133015953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,128,0.013754666845003764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,128,0.02672213315963745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,6144,0.17506880760192872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,64,0.012742400169372559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,64,0.026859732468922932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,3072,32,0.013380266229311624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,3584,0.10866453647613525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,32,0.026979200045267743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,3072,768,0.050426665941874185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,2560,0.08205866813659668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,1536,0.056387201944986975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,51200,1.3364341735839844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,65536,1.821401596069336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,768,0.038424531618754074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,65536,1.8159732818603516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,16384,0.4520458539326985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,16384,0.4671200116475423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,51200,1.417641576131185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,128,0.02363413373629252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,12288,0.34440959294637047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,12288,0.3551029205322266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,10240,0.26495679219563806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,51200,1.2880874633789063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,10240,0.31608851750691735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,8192,0.21394346555074056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,8192,0.27704426447550456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,8192,0.207367467880249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,7168,0.19526613553365071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,7168,0.2234879970550537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,3072,256,0.025836799542109174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,6144,0.16653547286987305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,6144,0.1865653355916341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,5120,0.1341375986735026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,5120,0.15790613492329914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,5120,0.12959252993265788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,4096,0.10869653224945068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,65536,1.5476351420084635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,4096,0.12928106784820556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,3584,0.0959712028503418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,3584,0.11454933484395344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,3584,0.09554666678110758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,3072,0.08310186862945557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,3072,0.09965759913126628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,12288,0.2929994583129883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,2560,0.0707594633102417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,2560,0.08834346930185953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,2048,0.05753920078277588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,7168,0.1761738618214925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,2048,0.07243093649546305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,1536,0.04461546738942464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,2048,0.06154559850692749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,16384,0.3858847935994466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,1536,0.050521600246429446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,1536,0.05899626811345419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,6144,0.15572907129923502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,1024,0.03173546592394511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,768,0.02618559996287028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,1024,0.04402773380279541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,512,0.01984106699625651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,768,0.03641173442204793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,512,0.030898133913675945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,256,0.013709867000579834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,256,0.02521066665649414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,10240,0.24486613273620605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,128,0.010250666737556457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,128,0.022107734282811483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,4096,0.10615039666493734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,64,0.008872532844543457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,64,0.021971199909845987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2560,32,0.009193600217501322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2560,32,0.022190932432810465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,768,0.03417280117670695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,3072,0.08351360162099203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,2560,0.07206079959869385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,65536,1.438145065307617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,1024,0.038819201787312824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,65536,1.3634357452392578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,65536,1.6823978424072266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,51200,1.08417599995931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,512,0.027589333057403565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,256,0.022115200757980347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,16384,0.3680800120035807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,12288,0.2559072017669678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2560,128,0.020323199033737183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,51200,1.3143637339274088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,16384,0.382366943359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,12288,0.32270612716674807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,10240,0.21471254030863443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,8192,0.18648959795633951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,10240,0.2738826751708984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,8192,0.21992534001668296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,7168,0.1603434721628825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,7168,0.15035732587178546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,7168,0.19564372698465984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,6144,0.13664533297220866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,6144,0.1317471981048584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,6144,0.17017280260721843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,5120,0.11538773377736408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,4096,0.0937280019124349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,5120,0.14446400006612142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,4096,0.1291317303975423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,3584,0.07884586652119954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,3584,0.1096501350402832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,16384,0.4370368003845215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,3072,0.06968639691670736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,3072,0.09340373675028482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,12288,0.251092259089152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,2560,0.05833066701889038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,10240,0.21057920455932616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,2560,0.07944640318552652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,51200,1.022707176208496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,2048,0.04747733275095622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,8192,0.17080853780110677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,2048,0.07110186417897543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,1536,0.0365664005279541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,1536,0.05422720114390055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,1024,0.02754666606585185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,1024,0.04103253285090129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,5120,0.11149439811706544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,1024,0.03385813236236572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,768,0.021409066518147786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,768,0.03391573429107666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,512,0.01694933374722799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,4096,0.09196693102518717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,512,0.0287445326646169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,512,0.023781333367029825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,256,0.011688533425331115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,256,0.02373653252919515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,3584,0.08155840237935384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,128,0.008842666943868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,128,0.024130133787790935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,128,0.017081600427627564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,64,0.007805866499741872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,3072,0.07145067056020102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,2048,32,0.008353066444396973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,64,0.024378667275110878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,2048,32,0.023975465695063272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,65536,1.136846923828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,2560,0.06127039988835653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,65536,1.5203871409098306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,2048,0.052434134483337405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,1536,0.04296106497446696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,768,0.02820906639099121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,16384,0.2633333365122477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,65536,1.1674421946207683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,51200,0.8134069442749023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,12288,0.20509759585062662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,16384,0.3858912150065104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,2048,256,0.018523732821146645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,16384,0.2869365374247233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,12288,0.3367583910624186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,10240,0.16913493474324542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,12288,0.21127786636352539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,51200,1.1896991729736328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,8192,0.13458986282348634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,10240,0.24613439242045082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,7168,0.11872533162434895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,8192,0.21648213068644204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,8192,0.15758612950642903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,6144,0.10155519644419353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,7168,0.1316874663035075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,5120,0.08675413131713867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,7168,0.17975893020629882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,6144,0.15552426973978678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,4096,0.0707914670308431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,5120,0.13272106647491455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,4096,0.11049706935882568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,3584,0.062338133653004967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,4096,0.07724160353342692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,3584,0.09598186810811361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,3072,0.08319040139516196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,2560,0.045269334316253663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,2560,0.07083626588185629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,3072,0.05518720149993896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,2048,0.03682773510615031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,2048,0.058789332707722984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,51200,0.8644608179728189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,1536,0.03123626708984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,1536,0.05040853420893351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,10240,0.18008747100830078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,1536,0.040720001856486006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,1024,0.020498132705688475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,1024,0.02754666606585185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,1024,0.03696533441543579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,768,0.01686613361040751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,768,0.030802132685979207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,6144,0.1128874699274699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,512,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,512,0.02563520073890686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,512,0.02018773357073466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,256,0.009444266557693481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,5120,0.09598293304443359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,256,0.021976532538731892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,3584,0.06997120380401611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,256,0.01586133340994517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,3072,0.06053866545359293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,64,0.0064085334539413456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,128,0.020148267348607383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,32,0.0066453332702318835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,64,0.019746132691701255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1536,32,0.019346133867899577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,2560,0.05267200072606405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,65536,0.6693461100260417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,51200,0.5400213241577149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,65536,0.8754922866821289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,16384,0.17384959856669108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1536,128,0.0075434664885203045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,65536,1.3134282430013022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,12288,0.12953813076019288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,16384,0.35496107737223304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,2048,0.045026131470998126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,10240,0.1095253308614095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,51200,1.0454421361287436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,12288,0.25835413932800294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,10240,0.14669119517008464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,8192,0.08715306917826335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,10240,0.21920107205708822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,7168,0.07812480131785074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,8192,0.1791381359100342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,7168,0.15881600379943847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,6144,0.06750720342000326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,6144,0.13883412679036458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,768,0.024075732628504435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,5120,0.05617920160293579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1536,128,0.014493866761525472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,5120,0.11963626543680828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,51200,0.6749184290568034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,4096,0.04623253345489502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,4096,0.09706880251566569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,16384,0.22384959856669107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,12288,0.17213226954142252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,3584,0.04060693184534709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,3584,0.08576426506042481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,3072,0.03560426632563273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,8192,0.11771199703216553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,3072,0.07333119710286459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,2560,0.0298634668191274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,7168,0.10457173188527424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,2560,0.06165759960810343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,2048,0.02450986703236898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,2048,0.04929920037587483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,6144,0.09139626820882162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,1536,0.01960106690724691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,1536,0.038472533226013184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,5120,0.07629866600036621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,1024,0.014638933539390563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,1024,0.021924267212549843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,1024,0.03139839967091878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,768,0.012132267157236736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,768,0.026396799087524413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,512,0.009935999910036724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,512,0.022946133216222128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,4096,0.06209493478139242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,256,0.007429333527882893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,256,0.019705599546432494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,3584,0.05565866629282633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,256,0.012500266234079996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,128,0.0062496001521746315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,3072,0.04878186782201131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,128,0.018538665771484376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,128,0.01159999966621399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,64,0.005677866439024607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,1024,32,0.0059562668204307554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,64,0.01866133411725362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,1024,32,0.01830079952875773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,2560,0.042728531360626223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,2048,0.03582079807917277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,51200,0.4071530659993489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,65536,0.5288394610087077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,16384,0.14600213368733722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,1536,0.028471465905507403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,16384,0.3478421211242676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,65536,1.2393472035725912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,768,0.018717867136001588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,51200,0.9736202875773111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,10240,0.08573226928710938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,1024,512,0.01560640037059784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,12288,0.24243413607279457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,8192,0.06935893694559733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,10240,0.20610346794128417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,8192,0.16748266220092772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,7168,0.06088106632232666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,7168,0.14899306297302245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,12288,0.10100266933441163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,6144,0.0520138661066691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,6144,0.13051520188649496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,12288,0.1589951992034912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,5120,0.04405759970347087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,5120,0.11101653575897216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,65536,0.8182218551635743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,4096,0.03608746528625488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,16384,0.2100149313608805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,4096,0.09370026588439942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,3584,0.032024532556533813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,3584,0.08288319905598959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,10240,0.13827412923177082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,3072,0.028259199857711793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,3072,0.06915626525878907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,51200,0.6362047831217448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,6144,0.0839786688486735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,2560,0.0232149342695872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,7168,0.0973695993423462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,2560,0.05689706802368164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,2048,0.019434666633605956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,2048,0.04520639975865682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,1536,0.015357866883277893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,1536,0.03505280017852783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,8192,0.1102453311284383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,1024,0.011648000280062357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,1024,0.02702186703681946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,3072,0.044842668374379474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,768,0.010005333026250203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,768,0.024616533517837526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,4096,0.05771093368530274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,512,0.008012799918651581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,512,0.022181334098180135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,5120,0.07064747015635173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,256,0.006344533463319142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,256,0.020146133502324422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,1536,0.025111466646194458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,128,0.005197866757710775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,128,0.01792853275934855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,3584,0.051183998584747314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,64,0.004833066463470459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,2560,0.039553066094716385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,64,0.01767573356628418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,768,32,0.005032533407211303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,768,32,0.017166932423909508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,65536,0.35426133473714194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,1024,0.01996586720148722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,2048,0.03242986599604289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,51200,0.2856405258178711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,65536,1.172762680053711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,51200,0.9199487686157226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,51200,0.5145440101623535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,16384,0.09269440174102783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,16384,0.16762986183166503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,12288,0.07013440132141113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,16384,0.298690128326416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,12288,0.22729387283325195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,10240,0.05854293505350748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,10240,0.19240105946858724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,10240,0.10874666372934978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,8192,0.0473034660021464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,8192,0.1565056006113688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,7168,0.04187413454055786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,7168,0.14072213172912598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,128,0.009486933549245197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,6144,0.036184533437093096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,256,0.010771200060844421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,6144,0.12193919817606609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,65536,0.6561973571777344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,5120,0.031795199712117514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,512,0.013912533720334372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,5120,0.10572373072306315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,4096,0.02516053318977356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,4096,0.04730879863103231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,4096,0.08546666304270426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,3584,0.022551467021306358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,3584,0.07442453702290854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,3072,0.019708800315856933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,3072,0.061741864681243895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,768,768,0.01642346680164337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,2560,0.016671999295552572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,2560,0.05104213158289591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,2560,0.032357333103815715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,2048,0.013762133320172629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,2048,0.040535465876261396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,2048,0.026343466838200887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,1536,0.011296000083287556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,1536,0.03188479940096538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,8192,0.08910613059997559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,1024,0.008798933029174805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,1024,0.026017065842946368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,1024,0.016505600015322367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,768,0.007577600081761678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,7168,0.07876693407694499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,768,0.02332800030708313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,512,0.00628053347269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,768,0.014360533157984415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,512,0.02064639925956726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,256,0.005246933301289876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,12288,0.13088746865590412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,256,0.01909866730372111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,128,0.004588800172011057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,6144,0.06777493158976236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,128,0.017373865842819212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,64,0.004271999994913737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,128,0.008516266942024231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,512,32,0.004387199878692627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,64,0.01664746701717377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,512,32,0.01695893406867981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,5120,0.05698133309682211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,65536,0.2152789274851481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,51200,0.17171306610107423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,3584,0.04230399926503499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,3072,0.037722667058308915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,16384,0.06346026659011841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,65536,1.0945035298665364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,16384,0.28943039576212565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,51200,0.8622943878173828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,12288,0.05954346656799316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,1536,0.021342933177947998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,12288,0.22047680219014487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,10240,0.052292267481486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,10240,0.18706347147623698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,8192,0.04024213155110677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,8192,0.15257493654886883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,512,0.011809066931406657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,7168,0.03602026700973511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,512,256,0.009469866752624512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,7168,0.13718080520629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,6144,0.030984532833099366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,6144,0.12010773022969563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,51200,0.4751584053039551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,65536,0.6063722610473633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,5120,0.02696533401807149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,5120,0.09938666820526124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,4096,0.020967467625935873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,16384,0.15714346567789714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,12288,0.11879680156707764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,4096,0.08094186782836914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,3584,0.019667200247446694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,3584,0.07072640260060628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,10240,0.10006293455759685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,3072,0.018026665846506754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,3072,0.05838826497395834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,8192,0.08026879628499349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,2560,0.01223466694355011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,2560,0.047399465243021646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,7168,0.07098453044891358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,2048,0.01074026624361674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,2048,0.03643093506495158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,6144,0.062301866213480626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,1536,0.008669867118199667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,1536,0.03003840049107869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,1024,0.006793599824110668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,1024,0.02467093269030253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,5120,0.052321068445841466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,768,0.006178133189678192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,4096,0.043150933583577473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,768,0.022091732422510783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,512,0.004923733572165171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,512,0.019355734189351402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,3072,0.034017066160837814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,256,0.004106666644414266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,256,0.01755733291308085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,2048,0.023647999763488768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,128,0.003798400113979975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,3584,0.03895680109659831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,128,0.016364799936612447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,64,0.0034656000634034475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,64,0.01593386630217234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,256,32,0.003643733263015747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,256,32,0.015894400080045064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,2560,0.028933332363764448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,65536,0.20797012646993002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,65536,1.0876747131347657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,51200,0.16058986981709797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,1536,0.019458132982254028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,1024,0.01458453337351481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,16384,0.058872532844543454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,51200,0.849291737874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,16384,0.2794111887613932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,12288,0.042431998252868655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,12288,0.21483413378397623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,512,0.01030399998029073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,10240,0.03845440149307251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,256,0.00819946676492691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,768,0.012449066837628682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,10240,0.18141333262125653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,8192,0.028989867369333906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,8192,0.14963839848836263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,7168,0.02602880001068115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,7168,0.13203306992848712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,256,128,0.007454933226108551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,6144,0.023215999205907188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,6144,0.11553813616434734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,65536,0.571889050801595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,5120,0.020526933670043945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,5120,0.0974133332570394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,51200,0.45439999898274736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,4096,0.01790720025698344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,4096,0.08011626402537028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,16384,0.15083519617716473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,3584,0.015223466356595359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,3584,0.06792213122049967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,12288,0.11506880124409993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,3072,0.012939733266830445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,3072,0.056149331728617344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,10240,0.09705599943796793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,2560,0.011569066842397054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,2560,0.04419306516647339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,8192,0.07852906386057536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,2048,0.009832533200581868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,2048,0.03507519960403442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,7168,0.0684384028116862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,1536,0.007769600053628286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,1536,0.02879679997762044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,6144,0.059395198027292886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,1024,0.006101333101590474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,1024,0.02395413319269816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,5120,0.05008746782938639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,768,0.0050453335046768185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,768,0.02131519913673401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,4096,0.04119893312454224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,512,0.004270933568477631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,512,0.020343466599782308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,3584,0.03711466789245606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,256,0.0036650667587916053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,3072,0.03246506651242574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,256,0.017484800020853678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,128,0.003305600086847941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,128,0.016591999928156534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,2048,0.022538665930430093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,64,0.0031850665807724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,64,0.015591466426849365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,128,32,0.00327360009153684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,1024,0.013622400164604188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,128,32,0.01563093364238739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,65536,0.19640960693359374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,2560,0.02728640039761861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,51200,0.15623680750528973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,1536,0.017844265699386595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,16384,0.05576106707255045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,16384,0.2824831962585449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,12288,0.04225493272145589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,65536,1.0826644897460938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,51200,0.8422922770182291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,10240,0.03684053421020508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,8192,0.03066026568412781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,12288,0.2122432072957357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,10240,0.17909332911173503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,7168,0.024142932891845704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,6144,0.02169706622759501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,8192,0.1460576057434082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,5120,0.01564479966958364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,7168,0.12967039744059244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,4096,0.01143893301486969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,5120,0.09484053452809652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,3584,0.009527466694513957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,4096,0.07564266522725424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,3072,0.008506666620572407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,768,0.011332266529401143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,3584,0.06480853160222372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,2560,0.008098133405049642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,3072,0.05362133185068766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,2560,0.042530135313669844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,1536,0.005624533196290334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,2048,0.03335040012995402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,1536,0.02794133424758911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,1024,0.023389865954717003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,768,0.00415786678592364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,768,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,512,0.0037493333220481873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,512,0.019054933389027914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,256,0.0033312000334262846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,256,0.01686186591784159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,128,0.00311253344019254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,128,0.016101333498954772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,64,0.002902399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,64,0.015400532881418863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,512,0.009341866771380106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,32,0.0031008000175158186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,32,0.015258666872978211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,64,6144,0.11279679934183758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,65536,0.19729706446329753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,51200,0.15617386500040692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,16384,0.055973335107167566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,2048,0.006791466474533081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,12288,0.04137706756591797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,64,1024,0.00481279989083608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,10240,0.03657386700312297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,16384,0.28039681116739906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,128,0.006612266600131989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,8192,0.030589866638183593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,12288,0.2125706672668457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,7168,0.027034666140874224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,10240,0.1791829268137614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,6144,0.023101866245269775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,8192,0.145798397064209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,5120,0.014651733636856078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,51200,0.8449504216512045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4097,128,256,0.007417599856853485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,4096,0.009324799974759419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,65536,1.0782527923583984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,6144,0.11209920247395833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,3584,0.008406399687131246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,5120,0.09463253021240234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,2560,0.006782933572928111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,4096,0.0754805326461792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,2048,0.006093866626421611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,2560,0.041682132085164386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,3072,0.05216639836629232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,1536,0.00517546683549881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,2048,0.032917332649230954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,768,0.003930666546026865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,1024,0.023145600159962972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,512,0.0035455999275048576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,768,0.021104000012079873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,256,0.0032149332265059153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,512,0.018969599405924478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,128,0.003011200080315272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,256,0.0164874662955602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,64,0.0028533334533373516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,128,0.01534293293952942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,32,0.002935466667016347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,64,0.015818666418393454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,32,0.015517866611480713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,7168,0.12881706555684408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,3072,0.00766293356815974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,3584,0.06472320159276326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4097,32,1024,0.004267733295758565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4097,32,1536,0.02768639922142029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,12288,4.94873046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,16384,6.363874308268229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,12288,8.95820821126302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,12288,5.8896937052408855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,16384,7.699154154459636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,16384,11.796294148763021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,10240,4.480332946777343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,10240,8.371972147623698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,8192,3.8195691426595055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,10240,5.65887451171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,8192,6.8215489705403645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,8192,4.076378631591797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,7168,2.870457712809245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,7168,5.351461283365885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,51200,22.396431477864585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,6144,2.468738047281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,6144,4.477191670735677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,6144,2.91438725789388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,5120,2.1404693603515623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,5120,2.370605977376302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,5120,3.830718994140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,7168,3.509402720133463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,4096,1.7685780843098957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,4096,3.01910400390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,4096,2.050161107381185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,3584,1.4594229380289714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,3584,2.7466049194335938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,3072,1.2917887369791665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,51200,27.409812418619794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,3072,2.315186055501302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,3072,1.7249813079833984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,2560,1.1805418650309245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,2560,1.921234130859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,2560,1.4665354410807292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,2048,0.9051178614298502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,1536,0.6861749649047851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,1536,1.3318965911865235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,2048,1.6623072306315105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,1024,0.5028885205586751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,1024,0.9890133539835612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,3584,1.8208309173583985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,768,0.41584211985270186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,768,0.6453450520833333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,1024,0.7208597183227539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,512,0.4828149477640788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,512,0.3220170656840006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,256,0.29096959431966146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,512,0.5036320050557455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,256,0.24247466723124184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,128,0.22461439768473307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,256,0.3981034596761068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,64,0.20838720003763833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,128,0.22252586682637535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,64,0.2179317315419515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,128,0.39042774836222327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,32,0.20358187357584634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,32,0.21290133794148763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,2048,1.0621856053670249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,1536,0.8298847834269205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,768,0.5678784052530925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,51200,38.40469156901042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,16384,5.484779866536458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,16384,9.281960042317708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,51200,18.020955403645836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,12288,8.134560139973958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,16384,5.882172648111979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,65536,20.68227742513021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,12288,4.043590291341146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,12288,4.756726582845053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,10240,3.375934855143229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,51200,21.33531494140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,10240,4.092921702067057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,10240,5.820887247721354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,8192,2.792407480875651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,8192,2.970682779947917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,8192,4.760978190104167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,7168,2.718636830647786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,51200,30.53931477864583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,7168,2.805230967203776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,65536,31.50181477864583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,7168,4.687690734863281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,6144,2.2717557271321613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,5120,1.604638926188151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,6144,3.7001993815104166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,4096,1.4341354370117188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,5120,3.0625770568847654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,4096,2.513175455729167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,3584,1.2257706960042318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,3584,2.336988830566406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,6144,2.345093282063802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,3072,0.9932608286539713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,3072,1.882035191853841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,2560,0.8813962936401367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,5120,1.923851776123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,2560,1.4721951802571616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,2560,1.0903252919514974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,4096,1.5102879842122396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,2048,0.6862986882527669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,2048,0.897761090596517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,2048,1.2596907297770181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,1536,0.5854506810506185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,1536,0.9117973327636719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,65536,38.56980387369792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,1536,0.7041973114013672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,1024,0.43525333404541017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,1024,0.5960512161254883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,3584,1.4185866038004558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,1024,0.5533397038777669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,768,0.5124757448832195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,512,0.3427466710408529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,512,0.26002666155497234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,256,0.1985674699147542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,768,0.3466464042663574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,128,0.17920640309651692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,128,0.2015999952952067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,3072,1.1730602264404297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,64,0.18700586954752604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,64,0.17905707359313966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,32,0.18288745880126953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,51200,32,0.17483839988708497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,51200,256,0.22831679979960123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,768,0.44831253687540695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,512,0.36186345418294275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,256,0.30872106552124023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,51200,128,0.28866666158040366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,16384,1.8222367604573566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,16384,3.2447680155436194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,16384,2.038382975260417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,65536,7.243312072753906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,51200,5.544263712565104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,12288,2.3034912109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,12288,1.4032736460367838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,51200,5.902565511067708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,65536,8.563467407226563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,12288,1.4733163197835286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,10240,1.17827517191569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,10240,1.2912202199300131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,10240,1.8897642771402996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,8192,0.8980138778686524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,8192,1.0417482376098632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,8192,1.5145674387613932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,7168,0.8365919748942057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,7168,1.4326207478841146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,6144,0.7539072036743164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,6144,1.142722193400065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,51200,9.34149169921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,6144,0.7684245427449544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,5120,1.0056416193644204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,5120,0.5833311716715495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,5120,0.6451797485351562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,4096,0.5474677403767904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,4096,0.484877872467041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,4096,0.7754261016845703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,65536,12.731651814778646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,3584,0.7209525426228841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,3584,0.4263413429260254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,3072,0.3481376012166341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,3584,0.5060117403666179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,3072,0.5802613576253255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,2560,0.3102837244669596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,2560,0.34189866383870443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,2048,0.2747786521911621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,2048,0.36713921229044594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,2560,0.47372481028238933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,1536,0.2893493334452311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,1024,0.20821119944254557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,1536,0.20002026557922364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,1536,0.26008853912353513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,1024,0.1726367950439453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,768,0.15834986368815104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,768,0.12335999806722005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,1024,0.19232640266418458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,768,0.1744128068288167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,512,0.1251050631205241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,512,0.09881707032521567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,256,0.07195839881896973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,256,0.07933226426442465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,128,0.05978560050328573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,256,0.13124266465504963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,128,0.073198930422465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,64,0.05561813513437906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,128,0.1047925313313802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,32,0.05618666807810465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,64,0.07251839637756348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,32,0.07120532989501953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,7168,0.8306453069051107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,3072,0.3787957191467285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,2048,0.2754240036010742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,512,0.12838613192240397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,16384,1.4129642486572265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,16384,2.3290079752604167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,51200,4.360963439941406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,12288,1.0154656092325847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,65536,5.680062866210937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,16384,1.6261428833007812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,12288,1.6552874247233074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,10240,0.9413045247395834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,10240,0.9501536051432291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,10240,1.4241503397623698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,51200,7.136385091145833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,8192,0.7158666610717773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,8192,1.1849343617757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,7168,0.9368842442830404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,7168,0.6161951700846354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,65536,9.175135294596355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,7168,0.7147104263305664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,6144,0.5233440081278483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,65536,6.283495585123697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,6144,0.5501109441121419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,5120,0.4426901181538899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,51200,4.729531860351562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,12288,1.0871509552001952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,4096,0.5412970860799153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,4096,0.36654081344604494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,4096,0.44245866139729817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,6144,0.8214239756266275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,3584,0.5110431989034017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,3584,0.32082452774047854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,3584,0.3613802591959635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,3072,0.3273717244466146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,3072,0.39169174830118814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,5120,0.7254709243774414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,2560,0.3487541198730469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,2560,0.26170667012532556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,2048,0.2734560012817383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,2048,0.21779413223266603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,2048,0.24065279960632324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,1536,0.20963199933369955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,1536,0.1673269271850586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,1024,0.1476426601409912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,1536,0.19229332605997723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,1024,0.12126293182373046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,768,0.11807573636372884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,1024,0.14608319600423175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,768,0.10040000279744465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,512,0.08765119711558024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,768,0.12476159731547039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,512,0.07686293125152588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,256,0.054321066538492835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,512,0.10477759838104247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,256,0.0643946647644043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,128,0.04723946650822957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,128,0.0581930677096049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,64,0.04326933224995931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,3072,0.3043978691101074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,64,0.05915946563084921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,32,0.04366720120112101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,8192,0.7228448232014973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,32,0.05880853335062662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,5120,0.45357974370320636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,2560,0.24868267377217612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,256,0.08202346960703531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,128,0.07577599684397379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,51200,3.7406036376953127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,16384,1.202740224202474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,65536,4.787705485026041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,16384,1.9374314626057942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,65536,5.166771443684896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,12288,1.476593017578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,12288,0.8811040242513022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,12288,0.9791594823201498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,51200,5.770321146647135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,10240,1.1685653686523438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,10240,0.8165856043497721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,8192,0.5924032211303711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,8192,0.983135986328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,7168,0.5246005376180013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,65536,7.4603220621744795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,7168,0.851524289449056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,6144,0.48308054606119794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,6144,0.7155978520711263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,5120,0.4124170621236165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,16384,1.2090314229329429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,5120,0.5829973220825195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,51200,3.8007764180501304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,4096,0.33100694020589194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,4096,0.5066111882527669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,4096,0.37148160934448243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,3584,0.41725759506225585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,3584,0.3095232009887695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,3584,0.31522881189982094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,10240,0.7884394963582356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,3072,0.3709322611490885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,2560,0.2277344067891439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,2560,0.22943466504414878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,8192,0.6016277313232422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,7168,0.5317514737447102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,2048,0.23293654123942056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,2048,0.20127253532409667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,1536,0.18172799746195475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,1536,0.14943466186523438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,1024,0.12765119870503744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,6144,0.4662912050882976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,3072,0.24732586542765297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,1024,0.10648639996846516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,768,0.09724053541819254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,768,0.08870720068613688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,768,0.10011733373006185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,512,0.07476267019907633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,2560,0.321397336324056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,512,0.06985599994659424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,256,0.05017386674880982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,256,0.05668479998906454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,256,0.06996266841888428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,128,0.03991146485010783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,128,0.051614932219187414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,128,0.06406613190968832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,64,0.036064000924428304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,32,0.03685119946797689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,5120,0.38741013209025066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,64,0.05141119956970215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,32,0.051932799816131595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,3072,0.24332480430603026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,2048,0.17996907234191895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,1536,0.14628693262736003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,1024,0.1161141316095988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,512,0.0843658685684204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,65536,3.977372741699219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,16384,1.4924395243326822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,16384,0.9701258977254232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,12288,0.7565216064453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,12288,1.1644864400227866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,51200,3.091448465983073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,51200,3.3869748433430993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,65536,5.877437845865885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,12288,0.7952351888020833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,10240,0.615771738688151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,10240,0.9150613149007162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,51200,4.6354924519856775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,10240,0.662606938680013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,8192,0.7235477447509766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,8192,0.5460117340087891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,8192,0.5151136080423991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,7168,0.6474655787150065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,7168,0.44600534439086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,6144,0.40042985280354815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,6144,0.551797358194987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,6144,0.4246805191040039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,5120,0.3300842603047689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,5120,0.4670389175415039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,5120,0.33028373718261717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,4096,0.2808554649353027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,4096,0.3759071985880534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,3584,0.33329172134399415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,3584,0.24117439587910972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,3584,0.24242879549662272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,3072,0.2871434529622396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,3072,0.21512319246927897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,3072,0.2104853312174479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,2560,0.22691946029663085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,2560,0.182695468266805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,2048,0.18230187098185222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,2048,0.14862186113993328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,2048,0.16340160369873047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,16384,0.9723658879597983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,65536,4.264113108317057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,1536,0.14101546605428059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,1536,0.12030719916025798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,1536,0.12944533030192057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,1024,0.11882346471150715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,1024,0.10584853490193684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,768,0.07868266900380452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,768,0.07381866772969564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,768,0.10085120201110839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,512,0.06078399817148844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,256,0.04052160183588664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,512,0.06988053321838379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,128,0.032433066765467325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,256,0.049259734153747556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,256,0.0627669334411621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,64,0.02969920039176941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,128,0.05431893269220987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,128,0.04369599819183349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,64,0.04653546810150146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,32,0.030326400200525922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,32,0.045244801044464114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,512,0.07516372998555501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,4096,0.2588895956675211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,2560,0.1750197410583496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,7168,0.4397610664367676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,1024,0.09513493378957114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,65536,3.5770027160644533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,16384,0.8753952026367188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,16384,1.2645951588948567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,65536,3.9397631327311196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,51200,2.768187713623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,16384,0.9049237569173177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,12288,0.7105013529459636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,12288,0.9445322672526041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,51200,4.044696553548177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,65536,5.140444946289063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,10240,0.5811370849609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,10240,0.5749216079711914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,51200,2.9230016072591143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,10240,0.7975221633911133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,8192,0.6758538564046224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,8192,0.4593514760335286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,8192,0.4612544059753418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,7168,0.4134058634440104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,7168,0.5895754496256511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,6144,0.5118560155232748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,6144,0.34370880126953124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,5120,0.3044608116149902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,5120,0.3910293261210124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,5120,0.3212309201558431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,4096,0.3244874636332194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,3584,0.290882142384847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,3584,0.22961813608805337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,3072,0.25800746281941733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,3072,0.1984608014424642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,6144,0.3881781260172526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,2560,0.19685120582580568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,4096,0.24397865931193033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,12288,0.6579573313395183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,2560,0.1547327995300293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,2048,0.12933759689331054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,2048,0.17642560005187988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,1536,0.13670719464619954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,2048,0.13426240285237628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,1536,0.11783359845479328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,1024,0.09789760112762451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,1024,0.09046613375345866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,1024,0.09567573070526122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,768,0.06994132995605469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,7168,0.3841461181640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,768,0.06966933409372965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,512,0.05309546788533529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,512,0.055927467346191403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,512,0.06425493160883586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,256,0.0341322660446167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,256,0.045269334316253663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,256,0.05277760028839111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,3584,0.20790185928344726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,128,0.029818665981292725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,4096,0.23129599889119468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,128,0.04180906613667806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,64,0.027151999870936076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,64,0.04200213352839152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,32,0.04195733467737834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,3072,0.18275307019551595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,2560,0.1570730686187744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,32,0.027242666482925414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,1536,0.11051946481068928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,768,0.07583573659261068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,128,0.04823466539382935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,16384,1.1209067026774089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,65536,3.169031524658203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,51200,2.4306859334309894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,16384,0.7779978434244792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,51200,2.548806508382161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,16384,0.8479167938232421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,12288,0.8401856104532877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,65536,4.513240559895833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,12288,0.5688608169555665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,51200,3.5416918436686196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,12288,0.5977919896443684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,10240,0.5390080134073894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,10240,0.7320234934488933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,8192,0.4521077473958333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,8192,0.41484587987263993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,8192,0.53538023630778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,7168,0.3723402659098307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,6144,0.32778774897257484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,6144,0.4110378583272298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,5120,0.3332042694091797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,5120,0.27212587992350257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,5120,0.27182826995849607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,4096,0.29391253789265953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,4096,0.21976319948832193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,3584,0.25736640294392904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,3584,0.19057812690734863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,3584,0.2042527993520101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,3072,0.19870506922403972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,7168,0.44989013671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,3072,0.17718186378479003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,3072,0.15900479952494304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,2560,0.167357873916626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,2560,0.14602773984273273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,2560,0.14274880091349285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,2048,0.13612373669942218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,1536,0.10541120370229085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,2048,0.11954773267110189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,65536,3.438056437174479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,1536,0.09695466359456381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,1536,0.0987669308980306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,1024,0.07477119763692221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,1024,0.09605653285980224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,768,0.0603007992108663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,768,0.0785269339879354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,768,0.06736746629079184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,512,0.05739733378092447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,512,0.05075093507766724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,256,0.030126933256785077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,256,0.04118080139160156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,256,0.04535040060679118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,128,0.025675733884175617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,10240,0.4759690602620442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,128,0.03789653380711873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,128,0.04916906754175822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,64,0.04054293235143026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,32,0.025889066855112712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,32,0.03905813296635945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,7168,0.3351658821105957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,6144,0.2880031903584798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,4096,0.2015381336212158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,64,0.023808000485102336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,2048,0.11618560155232746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,1024,0.07429013252258301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,512,0.05466026862462362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,16384,0.9201898574829102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,65536,2.826460774739583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,65536,2.7774037679036456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,16384,0.7072085062662761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,51200,2.1454058329264325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,51200,3.0475797017415363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,65536,3.7908991495768225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,12288,0.536411730448405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,12288,0.6607701619466145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,16384,0.6670207977294922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,51200,2.124562072753906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,10240,0.5707786560058594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,8192,0.38232428232828775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,10240,0.43594986597696944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,10240,0.46756798426310225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,8192,0.47159786224365235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,7168,0.4027669270833333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,6144,0.3305493354797363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,7168,0.3198261260986328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,6144,0.2875765482584635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,7168,0.3283893267313639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,6144,0.2797343889872233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,5120,0.22081920305887856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,5120,0.2469045321146647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,3584,0.20073067347208656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,4096,0.2024672031402588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,3584,0.17869866689046224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,3072,0.17308905919392903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,3072,0.1556821346282959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,2560,0.1438399950663249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,5120,0.28233814239501953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,4096,0.22234026590983072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,2560,0.12515520254770915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,2048,0.11663680076599121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,2560,0.11786879698435467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,2048,0.10565653642018635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,1536,0.09070933659871419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,2048,0.10126720269521076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,1536,0.08647039731343588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,1024,0.06447466611862182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,1536,0.08101973533630372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,1024,0.06706026395161947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,768,0.051387735207875575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,1024,0.06540586551030478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,512,0.03987413247426351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,768,0.05804373423258463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,512,0.04787520170211792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,256,0.02804479996363322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,12288,0.4889055887858073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,8192,0.32487465540568033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,256,0.037623465061187744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,128,0.02241706649462382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,256,0.0426528016726176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,128,0.03392746845881144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,64,0.01997973322868347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,128,0.03625173171361287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,32,0.020195200045903524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,64,0.034007465839385985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,32,0.034409598509470625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,4096,0.17442560195922852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,3584,0.1559999942779541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,3072,0.14007466634114582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,65536,2.376418050130208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,768,0.05741333166758219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,512,0.0470634659131368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,51200,1.8568416595458985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,65536,3.011588287353516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,51200,2.4264874776204426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,16384,0.7003679911295573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,16384,0.6038378397623698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,12288,0.5238837242126465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,12288,0.48528105417887374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,51200,1.8325546264648438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,10240,0.4845098813374837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,10240,0.386026668548584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,8192,0.33850132624308266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,8192,0.36881707509358724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,8192,0.31102078755696616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,7168,0.303003724416097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,7168,0.29519573847452796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,6144,0.2747733434041341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,6144,0.26142932573954264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,5120,0.22805652618408204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,5120,0.21674133936564127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,5120,0.1928767999013265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,4096,0.1800554593404134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,4096,0.17775039672851561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,4096,0.15258666674296062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,3584,0.1557546615600586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,3584,0.15213866233825685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,65536,2.3485087076822917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,3072,0.13563839594523114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,3072,0.1526367982228597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,2560,0.12319146792093913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,2560,0.1215008020401001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,16384,0.5414058685302734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,2048,0.09345280329386393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,12288,0.40333973566691084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,2048,0.0957482655843099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,10240,0.33698453903198244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,1536,0.07247680028279622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,1536,0.07975893020629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,7168,0.2449194590250651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,1024,0.051725868384043375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,1024,0.059604267279307045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,6144,0.20928212801615395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,768,0.04155733187993367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,768,0.05042133331298828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,768,0.04693973461786906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,512,0.031982932488123575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,512,0.041946665445963545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,3584,0.12914133071899414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,256,0.022091732422510783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,256,0.03400213321050008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,256,0.03222613334655762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,3072,0.11441919803619385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,128,0.01832533280054728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,128,0.02966933250427246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,128,0.029487999280293782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,64,0.014753066500027976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,64,0.030218666791915892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,32,0.015367466211318969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,32,0.03057173291842143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,1024,0.054212268193562826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,2560,0.09873706499735514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,2048,0.0835530678431193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,1536,0.06936533451080322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,16384,0.6474709192911784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,65536,2.160413869222005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,51200,2.156266784667969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,65536,2.280510965983073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,512,0.038889598846435544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,65536,2.9045430501302083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,51200,1.6614261627197267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,51200,1.7919509887695313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,12288,0.5058815956115723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,12288,0.44652798970540364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,8192,0.3415893236796061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,10240,0.41226666768391923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,8192,0.30742934544881184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,7168,0.2863082567850749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,7168,0.2693098704020182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,7168,0.229095458984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,6144,0.24263253211975097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,16384,0.6204981486002604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,6144,0.20058239301045738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,6144,0.23825599352518717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,5120,0.20429654121398927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,5120,0.20620373090108238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,4096,0.16560746828715006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,4096,0.16562879880269368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,10240,0.4432266553243001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,3584,0.14585280418395996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,3584,0.14438400268554688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,3072,0.12626880009969074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,3072,0.12272000312805176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,16384,0.4935221354166667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,2560,0.11724906762440998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,12288,0.36854079564412434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,10240,0.3076767921447754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,2048,0.08780906995137533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,2048,0.09101226329803466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,8192,0.24544107119242348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,1536,0.06680533091227213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,1536,0.07373226483662923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,5120,0.1610314687093099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,1024,0.04878826538721721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,1024,0.05655573209126791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,2560,0.11406079928080241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,4096,0.13301546573638917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,768,0.040454399585723874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,768,0.04838186502456665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,512,0.029663999875386555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,3584,0.11966293652852375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,512,0.04119786818822225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,256,0.020337067047754922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,256,0.032433066765467325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,256,0.029444267352422077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,128,0.015331199765205384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,128,0.027371732393900554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,3072,0.10559253692626953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,64,0.014452266693115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,64,0.02873813311258952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,32,0.015565866231918335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,32,0.028894933064778645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,2560,0.09070719877878824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,2048,0.07684266567230225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,1536,0.06350400050481161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,1024,0.04939093192418416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,768,0.04368213415145874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,512,0.03515199820200603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,65536,2.0109397888183596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,65536,1.8315210978190106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,51200,1.7876789093017578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,128,0.026310400168100996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,16384,0.5068373362223307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,12288,0.3903125445048014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,16384,0.5188192049662272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,12288,0.44467840194702146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,16384,0.47166080474853517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,51200,1.5630208333333333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,10240,0.32292054494222006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,65536,2.315105183919271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,8192,0.27484267552693686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,10240,0.33061014811197914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,7168,0.24191679954528808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,8192,0.26908801396687826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,6144,0.19625280698140463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,6144,0.22240427335103355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,7168,0.23804160753885903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,5120,0.16313066482543945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,5120,0.17671359380086263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,12288,0.32373867034912107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,4096,0.1332746664683024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,51200,1.4077259063720704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,10240,0.26784000396728513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,4096,0.14931947390238445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,3584,0.11665706634521485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,3584,0.1322719971338908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,3584,0.10566506385803223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,8192,0.221236260732015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,3072,0.11075519720713298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,7168,0.19611093203226726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,2560,0.08567573229471842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,2560,0.09553493658701578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,2048,0.08125013510386149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,6144,0.16987733840942382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,1536,0.055173333485921225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,1536,0.06718719800313314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,3072,0.10287679831186931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,1024,0.0399946649869283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,1024,0.0518069346745809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,2048,0.07011733055114747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,768,0.03230186700820923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,768,0.04382719993591309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,768,0.03829760154088338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,512,0.02502826650937398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,5120,0.1441461404164632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,512,0.036433064937591554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,256,0.01780160069465637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,256,0.02980053424835205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,4096,0.12071146965026855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,128,0.013700266679128012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,128,0.02667199969291687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,128,0.02339199980099996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,3072,0.09280426502227783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,64,0.011834667126337687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,32,0.012516267100969949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,64,0.027464532852172853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,32,0.02574719985326131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,2048,0.06874026457468668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,1536,0.05605866511662801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,2560,0.07958613236745199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,1024,0.04423786799112956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,65536,1.8934197743733723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,65536,1.54103151957194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,65536,1.8569557189941406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,512,0.031430399417877196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,51200,1.4532075246175131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,256,0.025549866755803424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,16384,0.47567148208618165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,16384,0.49150721232096356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,16384,0.40038185119628905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,12288,0.3155850728352865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,10240,0.29056212107340496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,12288,0.38759574890136717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,10240,0.3100693384806315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,8192,0.21300692558288575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,8192,0.20832533836364747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,51200,1.4338240305582681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,8192,0.26913493474324546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,7168,0.23138027191162108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,7168,0.1790272076924642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,6144,0.16261760393778485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,6144,0.20041813850402831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,5120,0.1362826665242513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,5120,0.1613375981648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,5120,0.13035840193430584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,4096,0.11060480276743571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,4096,0.1084437370300293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,4096,0.13165226777394612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,3584,0.0976960023244222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,3072,0.0849290688832601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,3584,0.11984000205993653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,3072,0.10558506647745769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,3072,0.08519360224405924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,2560,0.07157653172810872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,2560,0.0902890682220459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,2048,0.05886186758677164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,2048,0.07359573046366373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,7168,0.18522987365722657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,1536,0.04489279985427856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,1536,0.05917653242746988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,12288,0.28146559397379556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,1024,0.03217173417409261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,51200,1.2153429667154947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,10240,0.24306987126668295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,1024,0.04572480122248332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,768,0.026789333422978717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,1024,0.04718933502833049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,768,0.03846720059712728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,512,0.022634667158126832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,768,0.033735466003417966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,512,0.03505386511484782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,256,0.015553067127863566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,256,0.028889600435892743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,6144,0.15018773078918457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,128,0.010541866223017376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,128,0.02295359969139099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,3584,0.09346026579538981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,64,0.009190400441487629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,64,0.023099732398986817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,32,0.009469866752624512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,32,0.022125866015752158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,2560,0.0701802651087443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,2048,0.06096533139546713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,1536,0.050240000089009605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,65536,1.4740245819091797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,65536,1.3343594868977866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,65536,1.6647883097330731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,51200,1.0312768300374349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,16384,0.33488534291585287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,512,0.027534933884938557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,16384,0.45200427373250324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,256,0.022264534235000612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,12288,0.25645012855529786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,10240,0.20917654037475586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,12288,0.2426016012827555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,12288,0.33410027821858723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,10240,0.2757024129231771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,128,0.020127999782562255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,8192,0.16924479802449544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,51200,1.330459721883138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,8192,0.22300480206807455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,8192,0.1708405335744222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,7168,0.14822719891866049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,7168,0.19630400339762372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,6144,0.1273045301437378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,7168,0.15087679227193196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,5120,0.10799466768900554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,6144,0.17191893259684246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,5120,0.1443072001139323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,4096,0.08770453135172526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,4096,0.1203925371170044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,3584,0.07740373611450195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,3584,0.07834986845652261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,3584,0.10678186416625976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,3072,0.06712533632914225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,3072,0.09317866961161295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,2560,0.05794026851654053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,2560,0.07962559858957927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,16384,0.3242922782897949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,2048,0.04848426580429077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,2048,0.06862293084462484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,51200,1.0506111780802407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,1536,0.04198506673177083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,2048,0.055574401219685876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,1536,0.058918400605519616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,1024,0.030342400074005127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,1024,0.044283731778462725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,5120,0.10946986675262452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,6144,0.128875732421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,768,0.021450666586558025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,768,0.03325759967168172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,512,0.016029866536458333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,512,0.02788693308830261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,3072,0.06927893161773682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,256,0.011518933375676473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,256,0.02285226583480835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,256,0.018730666240056357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,2560,0.06085333426793417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,128,0.008794666330019633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,128,0.0213536004225413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,64,0.007683200140794118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,10240,0.2034602642059326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,32,0.008070399860541026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,64,0.02127573291460673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,32,0.02076479991277059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,4096,0.08864640394846598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,65536,1.1453109741210938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,65536,1.4933354695638021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,1536,0.04274026552836101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,1024,0.0333407998085022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,51200,0.9394336064656577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,768,0.02800106604894002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,16384,0.2621781349182129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,16384,0.4055818557739258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,512,0.023025067647298177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,51200,0.8815776189168295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,51200,1.1658709208170572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,10240,0.17385919888814289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,12288,0.3012245178222656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,128,0.016832000017166136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,10240,0.2450943946838379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,8192,0.1348042647043864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,7168,0.11885013580322265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,8192,0.14074667294820148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,8192,0.19946026802062988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,6144,0.09988266626993815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,7168,0.18131839434305827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,6144,0.1120149294535319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,6144,0.1541418711344401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,5120,0.08424853483835856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,5120,0.13172159989674886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,12288,0.19856425921122234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,5120,0.09308693408966065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,4096,0.06886613368988037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,4096,0.10849813620249431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,3584,0.06038506825764974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,3584,0.09631786346435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,65536,1.0889898935953775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,3072,0.05673280159632364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,3072,0.09098347028096518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,16384,0.27042560577392577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,2560,0.04448853333791097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,2560,0.07107199827829996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,12288,0.20752107302347816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,2048,0.03582079807917277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,2048,0.05825066566467285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,10240,0.1758698622385661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,1536,0.02821226716041565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,1536,0.04575999975204468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,7168,0.12521599928538005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,1024,0.02063360015551249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,1024,0.03419093290964763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,1024,0.02741546630859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,768,0.016612266500790916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,768,0.02996480067571004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,512,0.012974933783213297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,512,0.025547732909520466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,512,0.01967039903004964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,256,0.00902506709098816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,256,0.0214303990205129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,4096,0.07529919942220052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,128,0.007287466526031494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,128,0.020062933365503945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,3584,0.06691093444824218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,64,0.006442666550477346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,64,0.01983573238054911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,32,0.0066655998428662615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,32,0.020041600863138834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,3072,0.05904106696446737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,65536,0.6779050827026367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,2560,0.05142826636632284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,2048,0.043605331579844156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,65536,1.3077301025390624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,1536,0.0358730673789978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,51200,0.5206325213114421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,16384,0.1687658627827962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,16384,0.3544351895650228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,12288,0.12800640265146893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,768,0.023655466238657632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,12288,0.26141759554545085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,256,0.015608533223470052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,51200,1.0291285196940103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,10240,0.10722986857096355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,128,0.01448853313922882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,8192,0.0868565320968628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,10240,0.21738880475362143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,7168,0.07529813448588053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,8192,0.1780896027882894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,7168,0.1579093297322591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,6144,0.06513493458429973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,6144,0.13772692680358886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,65536,0.8724224090576171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,5120,0.0582207997639974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,5120,0.12230400244394939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,16384,0.2218389352162679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,4096,0.045259733994801835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,51200,0.6566613515218098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,4096,0.09741439819335937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,3584,0.04023146629333496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,12288,0.17059520085652669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,3584,0.08597760200500489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,3072,0.034731733798980716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,3072,0.07464213371276855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,10240,0.14217066764831543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,2560,0.02914239962895711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,8192,0.11687573591868083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,2560,0.0617792010307312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,7168,0.10182613531748455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,2048,0.024306132396062215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,2048,0.04953600168228149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,2048,0.03550506830215454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,1536,0.0195850670337677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,1536,0.038659199078877764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,1536,0.028488532702128096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,1024,0.014255999525388082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,1024,0.03031253417332967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,768,0.012103466192881267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,6144,0.08783573309580485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,768,0.0265664001305898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,512,0.009788800279299419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,512,0.023077332973480226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,512,0.01562666694323222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,256,0.007396266857783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,256,0.019879466295242308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,5120,0.07453440030415853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,128,0.006100266675154368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,128,0.018552533785502114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,4096,0.061127467950185144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,3584,0.054883201917012535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,64,0.005432533224423727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,64,0.01835839947064718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,32,0.0056096002459526065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,32,0.017859200636545815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,3072,0.04813333352406819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,2560,0.04211733341217041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,65536,0.5283669471740723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,1024,0.021702400843302407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,768,0.019014400243759156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,51200,0.4476458549499512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,16384,0.1506058692932129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,65536,1.236780802408854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,256,0.012497066458066305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,12288,0.10010133584340412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,16384,0.34986985524495445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,51200,0.9638741175333658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,12288,0.2429877281188965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,10240,0.08346986770629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,10240,0.2060981273651123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,8192,0.06775573094685873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,8192,0.16887466112772626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,128,0.011543466647466024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,7168,0.05904320081075033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,8192,0.10962453683217366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,7168,0.14945813814798992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,6144,0.051210665702819826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,6144,0.13077226479848225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,5120,0.04331093231836955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,5120,0.11138880252838135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,5120,0.07014719645182291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,4096,0.03534719944000244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,4096,0.09133760134379068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,65536,0.8097567876180014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,3584,0.03187519907951355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,3584,0.08254720369974772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,51200,0.6361941019694011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,3072,0.02719893256823222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,16384,0.20990079243977866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,3072,0.06929279963175455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,2560,0.02345386743545532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,12288,0.1615349292755127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,2560,0.05690240065256754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,2048,0.019573332866032918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,2048,0.045193600654602054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,10240,0.13298026720682782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,1536,0.015026133259137472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,1536,0.03522026538848877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,1536,0.02495680054028829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,1024,0.011404800415039062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,1024,0.02693013350168864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,7168,0.09609386920928956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,768,0.009843200445175171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,768,0.025140267610549927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,6144,0.08197653293609619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,512,0.008005333443482716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,512,0.02209920088450114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,4096,0.05706666707992554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,256,0.006187733511130015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,256,0.01901866594950358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,256,0.010726400216420491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,128,0.005293866495291392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,128,0.017797333002090455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,3584,0.0511680006980896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,64,0.004729599754015604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,64,0.017395200332005818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,3072,0.04422080119450887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,32,0.017299199104309083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,2560,0.03978879849116008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,2048,0.03254826664924622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,65536,0.3568458557128906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,51200,0.2958047866821289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,1024,0.019691733519236247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,768,0.016410666704177856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,32,0.004926933348178864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,16384,0.09257493019104004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,65536,1.150829823811849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,12288,0.07118613719940185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,51200,0.5111157417297363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,512,0.013982933759689332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,16384,0.2970677375793457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,51200,0.9386175791422525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,10240,0.057251199086507165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,12288,0.2276298681894938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,8192,0.04638506571451823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,10240,0.11185279687245686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,10240,0.1918517271677653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,8192,0.08839999834696452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,8192,0.15794026056925456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,6144,0.0366538683573405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,7168,0.13990720113118488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,6144,0.12294933001200359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,5120,0.031014400720596313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,128,0.009610666831334432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,5120,0.10456746419270832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,4096,0.025518933931986492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,4096,0.0851733366648356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,4096,0.04763199885686238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,3584,0.02211306691169739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,3584,0.07420480251312256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,7168,0.04174400170644124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,65536,0.6508181254069011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,3072,0.01890986760457357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,3072,0.06262293259302774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,16384,0.17156267166137695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,2560,0.016293332974116007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,2560,0.05068693161010742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,12288,0.13072853088378905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,2048,0.013633066415786743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,2048,0.040157866477966306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,7168,0.07779626846313477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,1536,0.011040000120798747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,2048,0.02632533311843872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,1536,0.03170880079269409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,6144,0.06755839983622233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,1024,0.008666666348775227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,1024,0.026173865795135497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,768,0.0075552001595497135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,5120,0.0572927991549174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,768,0.023264000813166298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,512,0.006362666686375936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,512,0.020581332842508952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,256,0.005176533261934916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,256,0.018273067474365235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,3584,0.04238400061925252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,128,0.0044821331898371375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,128,0.017056000232696534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,3072,0.037468798955281574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,64,0.004133333265781402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,64,0.016790399948755898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,32,0.004214400053024292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,32,0.016335999965667723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,2560,0.03220799962679545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,65536,0.21871573130289712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,1536,0.02109653353691101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,1024,0.01644159952799479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,768,0.014205867052078247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,51200,0.1717354615529378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,16384,0.06272639830907187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,512,0.011872000495592753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,65536,1.0877557118733725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,12288,0.051259732246398924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,16384,0.2862527847290039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,12288,0.21795199712117513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,51200,0.8577045440673828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,10240,0.04699519872665405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,256,0.009459199508031209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,10240,0.18410666783650714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,8192,0.033163734277089435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,8192,0.15026240348815917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,128,0.00849173367023468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,7168,0.03105066617329915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,7168,0.06985173225402833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,7168,0.13406720161437988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,6144,0.02770986755688985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,6144,0.11687466303507488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,5120,0.025060266256332397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,5120,0.09900800387064615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,65536,0.5812576293945313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,4096,0.018768000602722167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,4096,0.07923519611358643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,16384,0.15578667322794598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,3584,0.017438934246699015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,51200,0.4639616012573242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,3584,0.06930346488952636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,3072,0.015827199816703795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,12288,0.11802133719126384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,3072,0.058074665069580075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,10240,0.09889492988586426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,2560,0.04632000128428142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,8192,0.0805023988087972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,2048,0.011141332983970641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,2048,0.03627200126647949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,6144,0.06021226644515991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,1536,0.009103999535242716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,1536,0.029132799307505293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,5120,0.05123306512832641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,1024,0.0070933332045873005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,1024,0.025357866287231447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,2560,0.013155200084050498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,768,0.006026666859785716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,768,0.02211839954058329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,4096,0.04152213335037232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,512,0.005037866532802582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,512,0.019514666001001994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,3584,0.03772906859715779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,256,0.004222933451334635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,256,0.01773759921391805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,3072,0.03304106593132019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,128,0.003735466549793879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,2560,0.028524800141652422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,128,0.015941333770751954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,64,0.003509333233038584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,128,0.007398400207360585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,32,0.003730133424202601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,64,0.016012799739837647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,2048,0.022951465845108033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,32,0.015618133544921874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,65536,0.20585920015970866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,1536,0.018644267320632936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,51200,0.16402346293131512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,1024,0.014406399925549826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,65536,1.0780160268147787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,16384,0.05990933179855347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,51200,0.4481642723083496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,768,0.012106666962305706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,12288,0.03982293208440145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,51200,0.8490250905354818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,16384,0.2786143938700358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,12288,0.11536107063293458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,12288,0.21382506688435873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,10240,0.034663466612497966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,8192,0.02791680097579956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,10240,0.18040000597635905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,8192,0.14744319915771484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,7168,0.02474986712137858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,512,0.010155733426411946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,7168,0.13109333515167237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,6144,0.022259199619293214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,6144,0.11423679987589519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,256,0.008220799763997396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,5120,0.017968000968297322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,5120,0.09616959889729818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,65536,0.5663370768229167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,4096,0.01066986620426178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,4096,0.07789866924285889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,3584,0.00944213370482127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,3584,0.06659520069758097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,16384,0.15084160168965657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,3072,0.008775466680526733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,3072,0.0549450675646464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,10240,0.09658453464508057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,2560,0.007660800218582153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,2560,0.04413653214772542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,8192,0.07811840375264487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,2048,0.006985599795977275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,2048,0.03469226757685344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,7168,0.0684991995493571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,1536,0.005849599838256836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,1536,0.02823253273963928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,6144,0.05931839942932129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,1024,0.005006933212280273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,1024,0.023543467124303184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,5120,0.04919999837875366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,768,0.004430933296680451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,768,0.02132693330446879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,4096,0.04046719868977865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,512,0.003885866701602936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,512,0.019207467635472618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,3584,0.03594880104064942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,256,0.0034805332620938623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,256,0.017500799894332886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,3072,0.030874667565027873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,128,0.003206400076548258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,128,0.016153599818547568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,2560,0.027011199792226152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,64,0.0030901332696278887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,64,0.01591146687666575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,32,0.0033215999603271483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,32,0.015442132949829102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,2048,0.021415466070175172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,65536,0.1944149335225423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,1536,0.017566933234532674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,51200,0.15431466102600097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,1024,0.013381333152453104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,16384,0.052218667666117345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,12288,0.04128426710764567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,768,0.011100799838701884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,16384,0.2796799977620443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,12288,0.2124234676361084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,10240,0.03539199829101562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,8192,0.02959360082944234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,7168,0.026761599381764728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,51200,0.845090103149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,10240,0.17931946118672687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,8192,0.14616959889729816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,6144,0.0217631995677948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,5120,0.014873600006103516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,65536,1.0737664540608725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,7168,0.1292191982269287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,4096,0.008756267031033833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,6144,0.11259840329488117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,3584,0.008088533580303193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,5120,0.09432640075683593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,2560,0.008071466783682505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,3584,0.06463893254597983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,3072,0.05370879968007406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,2048,0.007129600147406261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,1536,0.005562666555245717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,2560,0.04206186532974243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,2048,0.032731733719507855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,1024,0.004825599988301595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,768,0.004166399935881296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,1024,0.023625600337982177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,512,0.0037439999481042228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,768,0.020980266729990642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,256,0.003365333378314972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,512,0.0190175990263621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,128,0.0030933332939942675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,256,0.01664426624774933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,128,0.01612160007158915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,64,0.002937600016593933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,512,0.009292800227801006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,32,0.003020799905061722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,64,0.015481600165367126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,32,0.01514240006605784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,65536,0.19000426928202313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,256,0.007295999924341838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,51200,0.15054720242818195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,16384,0.05095040003458658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,128,0.006495999793211619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,3072,0.008496000369389852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,4096,0.07567359606424967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,12288,0.04020693302154541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,16384,0.27718718846639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,10240,0.03423146804173787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,8192,0.028207999467849732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,7168,0.025378133853276568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,12288,0.2113855997721354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,10240,0.17898346583048502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,6144,0.022792534033457438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,8192,0.14532052675882975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,65536,1.0714261372884115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,5120,0.01632533371448517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,7168,0.12810239791870118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,4096,0.008342400193214417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,51200,0.840178108215332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,6144,0.1117141326268514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,3584,0.007474133372306823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,3072,0.006954666475454967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,2560,0.006253866851329804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,5120,0.09466773668924967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,3072,0.052492801348368326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,2048,0.005542399982611338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,2560,0.04146986802419027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,2048,0.03325653274854024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,1536,0.004852266609668731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,1024,0.004194133480389913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,768,0.0038890667259693147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,1536,0.027237333854039508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,512,0.0035637333989143372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,768,0.021638399362564086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,256,0.0032298666735490165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,512,0.018923733631769815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,128,0.002898133297761281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,256,0.016665599743525186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,64,0.002841600030660629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,128,0.015511467059453329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,32,0.003102933367093404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,64,0.015197867155075073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,32,0.015343999862670899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,1536,0.028024532397588092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,4096,0.075437863667806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,3584,0.0642250657081604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,1024,0.023192532857259116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,12288,4.471338907877604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,16384,6.191286214192709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,16384,8.199024963378907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,12288,8.398729451497395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,16384,11.062032063802082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,12288,5.981618245442709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,10240,3.811539204915365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,10240,7.73715311686198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,8192,3.7614507039388023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,8192,4.456057739257813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,10240,4.989279174804688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,8192,6.361565653483073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,7168,2.8247060139973956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,51200,19.81746622721354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,7168,5.1509953816731775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,6144,2.403478495279948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,7168,3.4304832458496093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,6144,2.874155680338542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,6144,4.210722096761068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,5120,2.0307050069173176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,5120,2.357085927327474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,5120,3.5266688028971354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,4096,1.5401610056559245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,4096,1.9510645548502605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,4096,2.8491785685221354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,3584,1.4269823710123697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,3584,2.5259241739908855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,3072,1.3322111765543618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,3072,2.1304959615071613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,3072,1.5061578114827474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,2560,0.995796267191569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,2560,1.7576981862386067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,2048,0.7749247868855794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,2048,1.4477845509847005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,2048,1.1658634185791015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,1536,0.6698005040486653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,3584,1.7482602437337238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,1024,0.46825278600056963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,1536,0.8761162439982095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,1024,0.7442837397257487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,768,0.37666025161743166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,768,0.5755925496419271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,1536,1.1019072214762369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,1024,0.7092992146809896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,512,0.428765869140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,512,0.2923167864481608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,256,0.23452372550964357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,256,0.22381013234456382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,128,0.20919787089029945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,128,0.2093770662943522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,2560,1.2437759399414063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,64,0.1957269350687663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,64,0.20525439580281576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,32,0.19158506393432617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,65536,32,0.2028010686238607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,768,0.5538997650146484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,512,0.4436629295349121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,256,0.3763008117675781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,128,0.3485504150390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,65536,51200,33.585750325520834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,65536,51200,36.50245361328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,16384,4.843226623535156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,16384,9.1173828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,16384,6.742262268066407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,51200,15.85516153971354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,12288,6.852509053548177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,12288,3.6564491271972654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,65536,20.788570149739584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,12288,4.70223134358724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,10240,3.2508522033691407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,10240,4.029933929443359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,10240,5.622555541992187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,8192,4.362256876627604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,8192,2.6117919921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,8192,3.1766827901204424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,7168,2.2247657775878906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,51200,28.18602701822917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,7168,4.208806355794271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,51200,27.105617268880206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,6144,2.0119764963785807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,6144,2.509288533528646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,6144,3.6416244506835938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,7168,2.6902496337890627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,5120,1.6723968505859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,5120,2.9529652913411457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,5120,1.9185461680094402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,4096,1.2780831654866538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,4096,1.5200298309326172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,3584,1.0781813303629557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,4096,2.311205291748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,3584,2.160869344075521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,3072,1.0060362497965494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,3072,1.7490826924641927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,2560,0.8407818476359049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,2560,1.4511924743652345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,65536,36.68045654296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,2048,0.7193024317423503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,2048,1.1783551534016927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,2048,0.8415967941284179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,65536,35.93616536458333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,1536,0.49459307988484696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,1536,0.7990421295166016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,1024,0.5557781219482422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,3584,1.2565471649169921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,1024,0.37188905080159507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,768,0.4540085474650065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,768,0.30794881184895834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,512,0.3180042584737142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,768,0.481444263458252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,256,0.19761066436767577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,512,0.2441802660624186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,256,0.18777813911437988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,3072,1.0864181518554688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,128,0.1731701374053955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,128,0.17514986991882325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,64,0.16192533175150553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,64,0.17165439923604328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,128,0.3057792027791341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,51200,32,0.15842666625976562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,51200,32,0.1681557337443034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,2560,0.9858997344970704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,1536,0.6400949478149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,1024,0.4776437441507976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,512,0.35050134658813475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,51200,256,0.29536320368448893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,16384,1.6803359985351562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,16384,2.900379689534505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,16384,2.0003018697102863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,65536,6.8167989095052075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,51200,5.412222798665365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,12288,2.0982602437337237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,12288,1.2943658192952474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,12288,1.457794189453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,10240,1.037871996561686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,65536,9.119307454427084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,51200,8.85883280436198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,10240,1.785256576538086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,10240,1.2666805267333985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,8192,0.852679443359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,7168,0.7500010808308919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,8192,1.4878016153971354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,7168,0.8935904184977213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,7168,1.2398272196451823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,6144,0.6487381617228191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,6144,1.116478983561198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,5120,0.5555829366048177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,5120,0.9105781555175781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,65536,11.473974609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,5120,0.6685984293619792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,4096,0.4606176058451335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,4096,0.7998783747355144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,51200,7.217949930826824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,4096,0.6790421168009441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,3584,0.6635391871134441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,3584,0.3796810785929362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,3584,0.5645301183064778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,3072,0.36350720723470054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,2560,0.4369301478068034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,3072,0.5354730606079101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,2560,0.3481034596761068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,2048,0.2536501407623291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,2048,0.37759040196736654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,1536,0.19973866144816083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,8192,0.922973887125651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,1536,0.2724661191304525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,2048,0.3405215899149577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,1024,0.14213760693868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,1024,0.22049387296040854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,768,0.1450837294260661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,768,0.1325642665227254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,768,0.15831146240234376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,512,0.12832852999369304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,512,0.08605226675669352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,512,0.15951573053995768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,256,0.06479893525441488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,256,0.07343466281890869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,6144,0.7126495997111003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,128,0.05777493317921957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,128,0.06663893461227417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,64,0.053257600466410315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,128,0.11559573014577229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,16384,32,0.05333546797434489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,32,0.0682474692662557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,3072,0.37741120656331384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,16384,64,0.06668372948964438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,2560,0.3155712127685547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,1536,0.2204223950703939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,1024,0.16820799509684245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,16384,256,0.10324906508127848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,16384,1.2940544128417968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,16384,2.164109802246094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,65536,5.03027089436849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,51200,4.0714047749837246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,12288,1.615329106648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,12288,0.9429429372151693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,10240,0.8166346867879233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,10240,1.3154708862304687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,65536,6.809957377115886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,51200,6.468385314941406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,65536,8.16837666829427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,10240,0.9379829406738281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,8192,1.0591050465901692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,8192,0.7014378865559896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,8192,0.7044106801350911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,16384,1.4468554178873698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,7168,0.592632548014323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,7168,0.9012938817342123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,6144,0.5067434628804525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,6144,0.8080682754516602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,5120,0.43665494918823244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,6144,0.6050581614176432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,5120,0.6526933034261068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,5120,0.479310925801595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,4096,0.3479050636291504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,4096,0.5621898651123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,4096,0.40572160085042314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,51200,4.984735107421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,12288,1.086187744140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,3584,0.48921918869018555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,3584,0.3151487986246745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,3072,0.2841962814331055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,3072,0.4386144002278646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,2560,0.25874346097310386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,2560,0.26444053649902344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,3072,0.3330826759338379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,2560,0.37692267100016275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,2048,0.27293332417805993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,1536,0.19574079513549805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,2048,0.2169343948364258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,1536,0.15767466227213542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,2048,0.23646933237711587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,1536,0.2179488023122152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,1024,0.14968105951944988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,1024,0.11256746451059978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,768,0.11151466369628907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,768,0.0957098642985026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,1024,0.15481600761413575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,512,0.08868800004323324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,256,0.05367253224054972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,512,0.0717087984085083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,256,0.06102506717046102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,256,0.08857920169830322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,128,0.04448533455530802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,128,0.05378559827804565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,128,0.07882453600565592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,64,0.04024426539738973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,12288,32,0.040275200208028154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,64,0.05493866602579752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,12288,32,0.05540693203608195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,7168,0.6190645217895507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,3584,0.3246453285217285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,768,0.1134229342142741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,12288,512,0.09512106577555338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,16384,1.0994047800699869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,16384,1.7941429138183593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,65536,4.554314676920573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,65536,5.761738586425781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,51200,3.4837259928385413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,12288,0.8079701105753581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,12288,1.2843007405598958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,51200,5.412080891927084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,10240,0.7248544057210287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,10240,1.0596864064534506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,65536,7.376437377929688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,8192,0.5804501215616862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,8192,0.8341834386189779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,8192,0.6043776194254558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,7168,0.4878517468770345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,7168,0.7121002833048503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,6144,0.6713749567667644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,6144,0.42087148030598953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,6144,0.4610591888427734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,5120,0.3592106819152832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,5120,0.5636917114257812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,16384,1.2101151784261068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,4096,0.41440426508585615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,3584,0.36542720794677735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,4096,0.35618133544921876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,12288,0.9011103947957357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,3584,0.2762432098388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,3584,0.3028618812561035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,10240,0.748745600382487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,51200,3.914650726318359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,3072,0.24197653134663902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,3072,0.35013866424560547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,2560,0.3136192003885905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,2560,0.20533973375956216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,2560,0.23359786669413246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,2048,0.24192426999409994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,2048,0.1893877347310384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,4096,0.316485341389974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,2048,0.18778880437215167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,1536,0.1798901398976644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,1536,0.1463637351989746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,1024,0.11256106694539388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,1536,0.18620373407999674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,1024,0.09891413052876791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,768,0.10538026491800945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,512,0.06771093209584554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,768,0.07955733140309652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,768,0.12916053136189778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,256,0.0430186669031779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,512,0.06873813470204672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,256,0.057010134061177574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,128,0.03848640124003093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,7168,0.5177952130635579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,128,0.04927146832148234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,64,0.035046398639678955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,10240,32,0.03523733218510945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,64,0.04943039814631144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,10240,32,0.050792535146077476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,5120,0.37731199264526366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,3072,0.24188159306844076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,1024,0.10955413182576497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,512,0.08062506516774495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,256,0.06611200173695883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,10240,128,0.061204266548156736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,65536,3.8589185078938804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,16384,0.9446581522623697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,16384,1.4684949239095053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,16384,1.0115552266438803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,65536,4.317464701334635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,12288,1.098471450805664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,12288,0.7544362386067708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,65536,5.710893758138021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,51200,2.9605641682942707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,10240,0.630297597249349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,12288,0.779310925801595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,10240,0.9331071853637696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,8192,0.7144522349039714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,8192,0.4847839991251628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,51200,4.519935099283854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,7168,0.6516031901041667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,7168,0.42917760213216144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,7168,0.4568021456400554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,6144,0.5244490623474121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,6144,0.3945119857788086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,6144,0.4065834681193034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,5120,0.4416063944498698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,5120,0.31862611770629884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,4096,0.2644522666931152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,4096,0.3640512148539225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,4096,0.28046293258666993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,3584,0.30787518819173176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,3584,0.22902612686157225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,3072,0.2682431856791178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,3584,0.25138667424519856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,8192,0.5129674593607585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,3072,0.2117077350616455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,2560,0.16560746828715006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,2560,0.23151680628458657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,2560,0.18583146731058758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,2048,0.17993280092875164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,2048,0.14873387018839518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,2048,0.15344319343566895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,1536,0.1381173292795817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,1536,0.11816319624582927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,1024,0.10065600077311199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,1536,0.12540799776713055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,1024,0.08736960093180338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,768,0.077619202931722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,768,0.07372480233510335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,768,0.08373653093973796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,512,0.058974933624267575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,10240,0.6104949315388997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,512,0.05672533512115478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,256,0.041187198956807454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,256,0.04713173309961955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,256,0.061419733365376795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,128,0.031923200686772665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,51200,3.295954132080078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,128,0.0431658665339152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,64,0.032576000690460204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,8192,32,0.0301962673664093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,64,0.04320106506347656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,8192,32,0.04623039960861206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,5120,0.31707305908203126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,3072,0.20175360043843588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,1024,0.09294400215148926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,512,0.06855040391286214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,65536,3.3877855936686196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,16384,1.3040660858154296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,8192,128,0.051293865839640296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,16384,0.8612149556477865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,65536,4.88529052734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,65536,3.7469642639160154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,51200,2.6557024637858073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,12288,0.8603530883789062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,12288,0.6483381271362305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,51200,2.9112640380859376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,10240,0.544874636332194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,51200,4.004201507568359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,12288,0.6641504287719726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,10240,0.595861307779948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,10240,0.8058762868245444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,8192,0.4520330746968587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,8192,0.5881919860839844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,7168,0.3865130742390951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,7168,0.5586528142293294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,6144,0.4488085428873698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,6144,0.34086294174194337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,6144,0.39075094858805337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,5120,0.36444374720255535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,5120,0.2911135991414388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,4096,0.3007871945699056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,4096,0.24146879514058434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,3584,0.21287466684977213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,3584,0.22314666112263998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,3072,0.23346026738484701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,3072,0.18132373491923015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,2560,0.19107093811035156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,3072,0.19546880722045898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,2560,0.15738560358683268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,2048,0.1601930618286133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,16384,0.9254079818725586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,2048,0.12833173274993898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,1536,0.12029759883880616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,2048,0.17582613627115887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,1536,0.10174612998962403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,3584,0.26419413884480797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,1024,0.08483413060506186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,1024,0.07710826396942139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,768,0.06697813669840494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,768,0.06472106774648032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,768,0.07547626495361329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,512,0.05025173425674438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,512,0.05221653381983439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,2560,0.16424427032470704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,256,0.03430933157602946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,256,0.042520534992218015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,7168,0.3848191897074381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,8192,0.43898986180623367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,128,0.028413865963617963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,128,0.03946773211161296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,128,0.048509867986043294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,64,0.025540266434351606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,64,0.03931306600570679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,7168,32,0.026009599367777508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,7168,32,0.03941226800282796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,5120,0.2849408149719238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,4096,0.23231679598490396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,1536,0.11056746641794842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,1024,0.08568747043609619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,512,0.062226132551829016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,7168,256,0.05172799825668335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,65536,2.954888407389323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,51200,3.3449483235677087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,16384,0.9594378789265952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,16384,0.798138682047526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,65536,4.27228037516276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,51200,2.3924725850423174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,16384,0.8034730911254883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,12288,0.5609877268473308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,12288,0.8261045455932617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,51200,2.528607940673828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,10240,0.5021173477172851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,12288,0.625819714864095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,10240,0.6623296101888021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,8192,0.5524906794230143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,8192,0.4038997332255046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,8192,0.38915945688883463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,7168,0.3799359957377116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,7168,0.446995194753011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,6144,0.3740490595499674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,6144,0.29180800120035805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,6144,0.3130666732788086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,5120,0.25761173566182455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,5120,0.2473898728688558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,4096,0.25920853614807127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,4096,0.20481066703796386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,3584,0.23438186645507814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,3584,0.1791594664255778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,3584,0.20440319379170738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,3072,0.1906496047973633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,3072,0.17158506711324056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,65536,3.382557932535807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,3072,0.16312747001647948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,2560,0.1670965353647868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,2560,0.135863463083903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,2560,0.1611296017964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,2048,0.13143680095672608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,2048,0.12039786974589031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,5120,0.3347285270690918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,2048,0.11560853322347005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,1536,0.10801920096079509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,1536,0.092302934328715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,1024,0.07191466490427653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,1024,0.06904746691385905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,1536,0.10350080331166585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,768,0.058102401097615566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,1024,0.07960106531778971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,512,0.04534613291422526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,768,0.059395198027292886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,512,0.04947520097096761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,256,0.031514666477839154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,256,0.038701868057250975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,256,0.045109331607818604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,4096,0.20540800094604492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,64,0.021995733181635536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,128,0.03651626507441203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,32,0.023808000485102336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,64,0.036600534121195474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,6144,32,0.03770560026168823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,10240,0.47746985753377275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,7168,0.3340853373209635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,6144,128,0.024572799603144325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,65536,2.5845919291178387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,768,0.06416853268941244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,512,0.05393066803614298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,6144,128,0.04158613284428914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,51200,2.0599732716878254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,65536,3.465924326578776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,51200,2.835737609863281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,16384,0.7982229232788086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,16384,0.6442410786946614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,12288,0.6391818364461263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,12288,0.5341898600260417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,12288,0.5009183883666992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,10240,0.41546986897786453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,10240,0.526142946879069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,8192,0.35120105743408203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,10240,0.4472074508666992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,7168,0.40181652704874676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,7168,0.3028618812561035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,7168,0.31957120895385743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,6144,0.3466090520222982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,51200,2.1689002990722654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,6144,0.26543359756469725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,6144,0.2769002596537272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,5120,0.2884288152058919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,5120,0.23776639302571617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,4096,0.22511146863301595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,4096,0.2010965347290039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,4096,0.18914346694946288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,3584,0.19803306261698406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,3584,0.17127572695414225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,3072,0.17122027079264324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,3072,0.1429258664449056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,8192,0.4133002599080403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,65536,2.750952657063802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,3072,0.15278293291727701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,2560,0.13452480634053549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,2560,0.11552533308664958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,2048,0.11501013437906901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,2048,0.10180479685465496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,2560,0.14184533754984538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,1536,0.09071253140767416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,1536,0.09210986296335856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,1536,0.09142186641693115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,1024,0.06771840254465739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,1024,0.0655455986658732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,1024,0.07132373650868734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,768,0.051268267631530764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,768,0.0563808004061381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,512,0.03798293272654216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,512,0.04487680196762085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,512,0.04915519952774048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,16384,0.6433301289876302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,256,0.026026666164398193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,256,0.037994666894276934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,256,0.04017066558202108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,128,0.021306665738423665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,128,0.03294613361358643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,128,0.03828266859054565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,64,0.0188810666402181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,64,0.033402665456136064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,5120,32,0.01986773411432902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,5120,32,0.032707200447718306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,8192,0.3315615971883138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,5120,0.21233386993408204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,3584,0.1570463975270589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,2048,0.09979413350423177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,65536,2.2888511657714843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,5120,768,0.05686933199564616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,65536,2.36093012491862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,65536,2.9803787231445313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,16384,0.5896938959757487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,16384,0.6808586756388346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,16384,0.5426997502644857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,51200,1.792118453979492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,51200,2.329060363769531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,12288,0.5379936218261718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,12288,0.46753705342610674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,12288,0.41337385177612307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,10240,0.45047998428344727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,8192,0.3577664057413737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,8192,0.314680544535319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,7168,0.31810665130615234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,6144,0.2550890604654948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,6144,0.2535157362620036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,6144,0.21932266553243002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,5120,0.22743679682413737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,5120,0.2007765293121338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,5120,0.19685866038004557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,4096,0.1694752057393392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,4096,0.16730453173319498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,10240,0.38830718994140623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,3584,0.15512107213338217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,3584,0.14317653973897299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,3584,0.1335594654083252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,7168,0.2900287946065267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,3072,0.13293440341949464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,3072,0.12437653541564941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,7168,0.26869014104207356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,2560,0.11247999668121338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,3072,0.11518186728159588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,2560,0.10619200070699056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,2560,0.09965120156606039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,2048,0.09268693129221597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,2048,0.08927146593729654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,1536,0.07159146467844645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,2048,0.08481600284576415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,1024,0.051284265518188474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,1536,0.07635626792907715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,1024,0.05733866691589355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,768,0.041790934403737386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,768,0.04956266482671102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,768,0.04799253145853678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,512,0.031931734085083006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,512,0.04002559979756673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,256,0.021977599461873373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,256,0.03214186628659566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,51200,1.6855018615722657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,8192,0.2692543983459473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,128,0.019641600052515664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,128,0.029332266251246138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,64,0.014408533771832785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,64,0.030333866675694782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,4096,32,0.016357333461443583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,4096,32,0.02990293304125468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,4096,0.14691413243611653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,10240,0.3379946708679199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,1536,0.06800106366475424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,1024,0.05353920062383016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,512,0.03895573218663533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,256,0.03176533381144206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,65536,2.562134297688802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,4096,128,0.029552000761032104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,16384,0.5800778706868489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,65536,2.083720525105794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,16384,0.5606730779012044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,16384,0.47411626180013017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,12288,0.48439572652180984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,51200,1.6196800231933595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,12288,0.41144746144612626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,51200,1.949178695678711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,10240,0.3644960085550944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,10240,0.3775007883707682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,10240,0.3145994822184245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,8192,0.3112757364908854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,8192,0.29006932576497396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,7168,0.2534026622772217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,8192,0.2672234535217285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,7168,0.22175893783569336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,7168,0.2563946723937988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,6144,0.2190165360768636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,6144,0.2295754591623942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,6144,0.1921354611714681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,5120,0.18429867426554364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,5120,0.18684906959533693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,4096,0.14736960728963217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,4096,0.15193600654602052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,4096,0.13754453659057617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,3584,0.13013439973195393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,3584,0.13133546511332195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,3584,0.12091626326243084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,3072,0.113809068997701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,3072,0.11528960069020588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,2560,0.09676693280537924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,2560,0.09740266799926758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,2048,0.07969706853230794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,2048,0.08267733256022135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,65536,2.0462154388427733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,1536,0.07264746824900309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,1536,0.07638933658599853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,12288,0.3642911911010742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,51200,1.5421588897705079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,1024,0.04735573530197144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,1024,0.052366932233174644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,1024,0.06199146509170532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,768,0.038427734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,768,0.04432533184687297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,768,0.052738134066263834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,512,0.028655999898910524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,512,0.04407039880752563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,512,0.03675946791966756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,256,0.023489065965016685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,256,0.031542400519053146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,128,0.015352533260981242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,128,0.02765333255132039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,128,0.026732800404230754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,64,0.01304213305314382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3584,32,0.013542399803797404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,5120,0.16289599736531574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,64,0.027537065744400024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3584,32,0.02721280058224996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,3072,0.10483520030975342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,2560,0.0898144006729126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,2048,0.07643093268076578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,65536,1.9210474650065106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,1536,0.06266773144404093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,65536,2.24717534383138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,51200,1.5105642954508463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,51200,1.6893931070963544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,16384,0.49110612869262693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,12288,0.37829653422037757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,16384,0.4939200083414714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,12288,0.41043519973754883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3584,256,0.02902719974517822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,10240,0.32786451975504555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,12288,0.34827518463134766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,10240,0.31743361155192057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,8192,0.24879786173502602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,7168,0.2295029322306315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,8192,0.28103679021199546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,7168,0.23368426958719887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,6144,0.19715946515401203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,6144,0.2005770683288574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,6144,0.17912319501241047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,5120,0.1596351941426595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,5120,0.17231787045796712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,4096,0.1292202631632487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,4096,0.14161492983500162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,65536,1.7585355122884114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,3584,0.12591893672943116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,3584,0.13293973604838055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,51200,1.3458901723225911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,3072,0.10460586547851562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,3072,0.11690026919047039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,16384,0.4313205401102702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,2560,0.08592320283253987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,3072,0.10454933643341065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,2560,0.08515733083089193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,2560,0.09335359732309977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,2048,0.07300693194071452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,2048,0.07921173572540283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,8192,0.22085332870483398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,1536,0.055029332637786865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,7168,0.19282453854878742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,1024,0.03923306862513225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,1024,0.04930026531219482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,1024,0.04337600072224935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,768,0.03209173281987508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,768,0.042045867443084715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,768,0.03853226502736409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,5120,0.14337706565856934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,512,0.024780799945195518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,512,0.03555733362833659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,512,0.03138346672058105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,256,0.01755519906679789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,4096,0.11762346426645916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,256,0.029821866750717164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,128,0.01461013356844584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,256,0.025754666328430174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,10240,0.27136640548706054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,128,0.026200532913208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,64,0.012344533205032348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,3072,32,0.012618666887283326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,64,0.02616426746050517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,32,0.025814400116602583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,3072,1536,0.06393813292185466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,3584,0.10615146954854329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,65536,1.7453088124593097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,2048,0.06802879969278972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,51200,1.3181407928466797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,1536,0.056150400638580324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,16384,0.44077545801798507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,16384,0.44400746027628585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,12288,0.302020263671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,3072,128,0.022871466477711995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,65536,1.708617655436198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,12288,0.34992427825927735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,10240,0.2490506649017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,8192,0.20112533569335939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,10240,0.2428991953531901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,10240,0.28197971979777015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,7168,0.17495466868082682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,8192,0.1987946669260661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,8192,0.23797547022501625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,6144,0.15251946449279785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,7168,0.20274453163146972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,6144,0.1490773359934489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,6144,0.1800490697224935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,5120,0.12702720165252684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,5120,0.15024852752685547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,51200,1.3660725911458333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,4096,0.10330666700998943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,4096,0.1227946678797404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,3584,0.09106880029042562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,3584,0.10834666887919109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,65536,1.5272202809651694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,3584,0.09386879603068034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,3072,0.09018879731496175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,3072,0.09442026615142822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,2560,0.06720000108083089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,2560,0.08135786851247152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,2560,0.06961386998494466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,2048,0.055097599824269615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,2048,0.06915413538614909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,16384,0.36964054107666017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,1536,0.043703468640645345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,51200,1.2018816630045572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,1536,0.0575925350189209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,1024,0.03266026576360066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,1024,0.04295039971669515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,5120,0.12598079840342205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,768,0.02443306644757589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,4096,0.10423040390014648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,768,0.034842665990193686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,512,0.018962132930755615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,512,0.02959360082944234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,12288,0.2795146624247233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,256,0.012999467055002847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,3072,0.08192426363627116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,256,0.02350826660792033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,256,0.02206826607386271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,128,0.009905067086219788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,128,0.022664533058802287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,64,0.008660266796747845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,64,0.021810134251912437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2560,32,0.00890880028406779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,7168,0.17339413960774738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2560,32,0.0216757337252299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,2048,0.059704534212748205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,1536,0.049318401018778484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,1024,0.03815679947535197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,65536,1.4098912556966146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,51200,1.096951421101888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,65536,1.5722933451334635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,16384,0.37653013865152996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,51200,1.2254815419514975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,12288,0.24118506113688148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,768,0.033367466926574704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,16384,0.4032149314880371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,16384,0.3778271993001302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,10240,0.20100480715433755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,12288,0.31541334788004555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,10240,0.28465919494628905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,8192,0.16280959447224935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,8192,0.22556692759195962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,128,0.019972266753514607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,7168,0.145033597946167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,7168,0.18308374087015789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,7168,0.1544864018758138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,6144,0.1217962662378947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,6144,0.1648245334625244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2560,512,0.027090134223302205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,5120,0.10335253079732258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,5120,0.13594560623168944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,5120,0.10586453278859456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,4096,0.08406399885813395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,4096,0.11185493469238281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,65536,1.2788885752360026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,3584,0.07719573179880777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,3584,0.10520106951395672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,3584,0.08400320212046306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,3072,0.06473066806793212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,3072,0.08926080067952474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,3072,0.06933226585388183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,2560,0.054946132500966395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,12288,0.236680539449056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,51200,1.0241621017456055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,2560,0.07549333572387695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,2560,0.05908053318659464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,2048,0.05114560127258301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,2048,0.05065600077311198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,10240,0.20146133104960123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,1536,0.040139734745025635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,1536,0.0491104006767273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,1536,0.041815467675526935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,1024,0.037324798107147214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,8192,0.16322132746378581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,768,0.020197333892186482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,768,0.03137600024541219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,512,0.015520000457763672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,512,0.02695680061976115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,512,0.023141332467397056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,256,0.011008000373840332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,256,0.022425599892934165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,6144,0.12544000148773193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,128,0.008587732911109924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,128,0.02084266742070516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,64,0.007382399837176005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,64,0.020678400993347168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,32,0.00786133309205373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,2048,0.06304639975229899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,2048,32,0.020588799317677816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,2048,1024,0.02928000092506409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,4096,0.08706239859263101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,65536,1.0184256235758462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,65536,1.378875732421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,51200,0.7355146408081055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,16384,0.27372159957885744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,1024,0.03215786616007487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,768,0.0273087998231252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,16384,0.2616543928782145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,51200,1.1207168579101563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,16384,0.38974294662475584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,12288,0.17955199877421063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,10240,0.1506495952606201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,256,0.018371200561523436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,10240,0.22761386235555015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,12288,0.27249813079833984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,10240,0.17716693878173828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,8192,0.11926826635996501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,7168,0.10629226366678875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,8192,0.18513600031534833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,2048,128,0.01657600005467733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,6144,0.09192533493041992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,7168,0.1267616033554077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,7168,0.1642218589782715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,5120,0.07690880298614503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,6144,0.1439733346303304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,5120,0.12209813594818116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,5120,0.09308906396230063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,4096,0.06280213197072347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,3584,0.05533759991327921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,4096,0.10025066534678143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,3584,0.0882250706354777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,3072,0.04804906845092773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,3072,0.07693013350168863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,3072,0.05848533312479655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,2560,0.04031039873758952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,2560,0.06485439936319987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,65536,1.08962771097819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,6144,0.10541120370229085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,2048,0.035702399412790936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,2048,0.05309226512908936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,2048,0.04990080197652181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,1536,0.024949334065119424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,1536,0.04527253309885661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,1536,0.03514240185419719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,1024,0.019077332814534505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,1024,0.031351466973622635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,1024,0.02876586715380351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,768,0.014407466848691305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,768,0.023497599363327026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,768,0.028243199984232588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,512,0.0109333336353302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,512,0.023909332354863484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,256,0.007892266909281413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,256,0.02152000069618225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,51200,0.7856010437011719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,128,0.007287466526031494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,128,0.01994133392969767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,12288,0.20284479459126792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,128,0.014167466759681701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,64,0.0063285330931345625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,8192,0.1381589412689209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,64,0.019603200753529868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1536,32,0.006659199794133504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1536,32,0.01918720006942749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,4096,0.07395093441009522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,65536,0.6668405532836914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,3584,0.0661514679590861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,2560,0.0504746675491333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,51200,0.5656991958618164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,65536,1.2413312276204427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,65536,0.8161376317342123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,16384,0.18594986597696941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,12288,0.1389568010965983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,51200,0.9693461100260417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,16384,0.3214655876159668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,10240,0.10329066912333171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,51200,0.6381664276123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,12288,0.16286932627360026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,12288,0.2639669259389242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,8192,0.0826101303100586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,10240,0.20590933163960776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,8192,0.11233920256296795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,8192,0.17515093485514324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,7168,0.0735157330830892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,512,0.01950719952583313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,6144,0.06363733212153116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,7168,0.1497536023457845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,5120,0.05284266471862793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,6144,0.13071466286977132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1536,256,0.01562346617380778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,5120,0.07190186977386474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,4096,0.044011731942494706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,5120,0.11159466902414958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,3584,0.0382421334584554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,4096,0.05967146555582682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,4096,0.09072639942169189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,3072,0.03333866596221924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,3584,0.07960853576660157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,3072,0.06853973070780436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,3072,0.047229866186777755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,2560,0.028125866254170732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,2560,0.05683626731236776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,2048,0.023388799031575522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,2048,0.046326398849487305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,2048,0.034254932403564455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,1536,0.01871253252029419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,1536,0.03625599940617879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,1536,0.027677865823109944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,1024,0.013916800419489542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,1024,0.029106134176254274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,768,0.011748266220092774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,768,0.02542080084482829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,6144,0.08570133050282797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,512,0.009385599692662557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,512,0.02224640051523844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,512,0.015549866358439126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,256,0.007131733496983846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,256,0.01977919936180115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,3584,0.053471998373667395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,128,0.00613973339398702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,128,0.01849173307418823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,16384,0.2125333309173584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,64,0.005317333340644837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,64,0.01815999945004781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,1024,32,0.005727999905745188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,1024,32,0.017927465836207072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,10240,0.137934939066569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,7168,0.09855360190073649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,65536,0.48100798924763993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,51200,0.4013717333475749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,65536,1.1595157623291015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,1024,0.021112533410390218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,16384,0.12036800384521484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,51200,0.9547882715861002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,768,0.018947199980417887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,12288,0.09041066964467367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,16384,0.2982240041097005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,12288,0.22736852963765464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,256,0.01237546702226003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,10240,0.07596266269683838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,2560,0.040453334649403885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,10240,0.19225279490152997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,8192,0.06241600116093954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,8192,0.1572725296020508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,7168,0.05519573291142782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,7168,0.13977386156717936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,1024,128,0.011152000228563944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,6144,0.04838186502456665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,65536,0.7713173548380534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,6144,0.12314240137736003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,5120,0.04082026481628418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,6144,0.0821664015452067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,5120,0.10546560287475586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,4096,0.03342080116271973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,4096,0.08534506956736246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,51200,0.6125951766967773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,3584,0.029286400477091475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,16384,0.20237973531087242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,3584,0.07514879703521729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,12288,0.15218559900919598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,3072,0.025358933210372924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,3072,0.06410346825917562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,10240,0.12785920302073162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,2560,0.02128960092862447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,2560,0.05233706633249918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,8192,0.10540160338083904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,2560,0.038098132610321044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,2048,0.017886932690938315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,2048,0.041713066895802814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,2048,0.03115306695302327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,1536,0.033802668253580734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,7168,0.09232959747314454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,1024,0.010877866546312969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,1024,0.027081600824991864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,1536,0.014317867159843446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,768,0.009212799866994222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,768,0.02413653333981832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,5120,0.06802240212758383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,512,0.007570133109887441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,512,0.021259733041127524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,4096,0.05530773401260376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,256,0.006118399898211161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,256,0.019346133867899577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,128,0.005242666602134705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,128,0.0176746666431427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,3584,0.049380266666412355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,64,0.004693333307902018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,64,0.017270400126775106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,768,32,0.004964266717433929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,3072,0.04312106768290202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,768,32,0.01700053413709005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,65536,0.3438410758972168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,1536,0.024292266368865965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,51200,0.27574612299601237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,1024,0.019042134284973145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,65536,0.6657589594523112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,65536,1.0930581410725912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,16384,0.08926826318105062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,16384,0.28124478658040364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,12288,0.06800106366475424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,51200,0.8665482838948568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,12288,0.21578559875488282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,10240,0.05693759918212891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,768,0.016107733050982155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,10240,0.18255680402119953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,8192,0.04609280029932658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,8192,0.14936960538228353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,256,0.010613333185513813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,8192,0.08440106709798177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,7168,0.04149119853973389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,7168,0.07463466326395671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,7168,0.13272426923116049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,128,0.009273599584897358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,6144,0.03595946629842122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,5120,0.03064533273379008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,6144,0.06455573240915934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,6144,0.11578453381856282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,4096,0.024581333001454674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,5120,0.09871679941813151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,4096,0.08022933006286621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,4096,0.04553706645965576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,3584,0.021605332692464195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,3584,0.06933120091756186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,3072,0.018969599405924478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,3072,0.058721065521240234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,768,512,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,2560,0.016149333119392394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,2560,0.0476032018661499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,2560,0.03033813238143921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,2048,0.0135861337184906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,2048,0.03753279844919841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,2048,0.02507200042406718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,1536,0.011116799712181092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,1536,0.030548266569773358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,1536,0.020343466599782308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,1024,0.00864533285299937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,16384,0.16437333424886066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,51200,0.48941545486450194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,1024,0.02540266712506612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,768,0.007490133245786031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,768,0.022940800587336222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,768,0.013991467157999673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,512,0.006309333443641663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,512,0.020408533016840615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,256,0.005245866874853769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,256,0.018196266889572144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,12288,0.12500693003336588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,128,0.0048885335524876915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,10240,0.1032960017522176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,128,0.017197867234547935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,128,0.008152533570925396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,64,0.00443200021982193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,512,32,0.004515199859937032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,64,0.017299199104309083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,512,32,0.016521599888801575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,65536,0.20317546526590982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,1024,0.01607039968172709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,5120,0.0550495982170105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,51200,0.16343785921732584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,256,0.009361066420873006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,3584,0.04125226736068725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,16384,0.06317333380381265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,3072,0.03631680011749268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,51200,0.44795732498168944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,16384,0.27139307657877604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,12288,0.058118398984273276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,51200,0.8056170781453451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,12288,0.20869973500569663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,10240,0.04747733275095622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,10240,0.09397119681040446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,10240,0.1783786614735921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,8192,0.037656533718109134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,8192,0.14625600179036458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,7168,0.035930665334065755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,65536,1.0283487955729167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,7168,0.1284063975016276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,512,512,0.01157866617043813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,6144,0.03110719919204712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,6144,0.11314559777577718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,5120,0.02749119997024536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,5120,0.09381120204925537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,65536,0.570419184366862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,4096,0.020901334285736085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,4096,0.07605653603871664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,3584,0.018616533279418944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,16384,0.14782826105753583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,3584,0.06568640073140462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,3072,0.016438399751981102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,12288,0.1124341328938802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,3072,0.05461333195368449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,2560,0.012216533223787945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,2560,0.04355413516362508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,2560,0.026881066958109538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,2048,0.010506666700045268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,8192,0.07672106424967448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,2048,0.03451519807179769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,1536,0.00902186632156372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,1536,0.028602667649586994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,6144,0.058721065521240234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,7168,0.06673493385314941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,1024,0.023676800727844238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,1024,0.01400106648604075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,768,0.005773866673310598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,5120,0.05083093245824179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,768,0.02135466734568278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,512,0.004901333153247834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,512,0.01939520041147868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,1024,0.007284266750017802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,256,0.004014933357636133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,4096,0.04112319946289063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,256,0.017534933487574258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,128,0.0038730666041374207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,128,0.01607253352801005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,3584,0.03675946791966756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,64,0.003517866631348928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,64,0.01581653356552124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,3072,0.03223466674486796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,256,32,0.003573333223660787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,256,32,0.016225066781044007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,65536,0.19777599970499676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,2048,0.02241599957148234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,51200,0.14692586263020832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,1536,0.01862293283144633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,65536,1.013100814819336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,768,0.012145066261291504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,16384,0.05454399983088175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,512,0.010265599687894185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,12288,0.04363306760787964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,16384,0.26361494064331054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,12288,0.10851946671803792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,12288,0.2027029355367025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,10240,0.0343722661336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,10240,0.17135252952575683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,8192,0.0265664001305898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,8192,0.13987946510314941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,256,0.00827519992987315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,7168,0.024445867538452147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,256,128,0.00709440012772878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,51200,0.7951413472493489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,7168,0.12338666915893555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,6144,0.021734400590260824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,6144,0.05591893196105957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,5120,0.015583999951680503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,5120,0.09072746435801188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,65536,0.5396757125854492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,4096,0.011372799674669903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,51200,0.4249919891357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,4096,0.0722378651301066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,3584,0.010294399658838908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,6144,0.10706559816996257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,3584,0.061197865009307864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,3584,0.03540159861246745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,3072,0.00925973355770111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,3072,0.05003413359324137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,2560,0.008147199948628742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,16384,0.14317760467529297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,2560,0.040278398990631105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,2560,0.02579413255055745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,2048,0.007281066477298736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,2048,0.03350720008214315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,10240,0.0913802703221639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,1536,0.006086400151252747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,1536,0.027185066541035967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,8192,0.07376320362091064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,1024,0.005122133096059163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,1024,0.02286293307940165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,7168,0.06427520116170247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,768,0.00458133320013682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,768,0.02074986696243286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,768,0.011064533392588298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,512,0.004137599964936575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,512,0.01844266653060913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,5120,0.04727360010147095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,256,0.0037685332198937735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,256,0.016703999042510985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,4096,0.03914560079574585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,128,0.003401600072781245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,128,0.01576959987481435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,128,0.006404266754786174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,64,0.003230933348337809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,3072,0.030109866460164385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,64,0.015593600273132325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,128,32,0.0032042667269706728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,128,32,0.015530666708946228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,65536,0.18598400751749675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,51200,0.14749013582865397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,2048,0.021107200781504312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,16384,0.05083946784337362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,12288,0.039851733048756915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,1536,0.01697280009587606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,16384,0.26449492772420247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,10240,0.03421440124511719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,65536,1.0074858983357748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,8192,0.029029333591461183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,12288,0.1999519983927409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,7168,0.023165865739186605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,10240,0.16950186093648273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,1024,0.012731732924779257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,6144,0.020606933037439983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,51200,0.7863210678100586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,8192,0.13774506251017254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,7168,0.12202239831288655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,4096,0.010974933703740437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,6144,0.10633066495259602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,5120,0.0886613368988037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,3584,0.009172266721725464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,3072,0.008318933347860973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,4096,0.07034986813863119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,2560,0.007559466858704884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,3584,0.05975786844889323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,2048,0.0065994665026664736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,3072,0.04951466719309489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,2560,0.0392576018969218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,1536,0.005598933498064677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,1024,0.0047189335028330484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,768,0.00415786678592364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,1536,0.026961066325505573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,1024,0.023694932460784912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,512,0.003764266769091288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,768,0.020677334070205687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,256,0.0033471999069054925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,512,0.018553600708643595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,128,0.003156266609827677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,256,0.016505600015322367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,64,0.003010133405526479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,128,0.01579839984575907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,32,0.0029781334102153777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,64,0.015548800428708395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,512,0.009284266829490661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,32,0.015121066570281982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,16384,0.04947733481725057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,65536,0.19087467193603516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,51200,0.14814292589823405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3841,128,256,0.007367466886838277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,12288,0.04118186632792155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,10240,0.032807467381159465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,8192,0.028109866380691528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,16384,0.26345599492390953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,10240,0.16929492950439454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,12288,0.20059520403544107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,7168,0.024576000372568765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,8192,0.13716799418131512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,6144,0.020639999707539877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,5120,0.01032533347606659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,7168,0.12163413365681966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,4096,0.008225066463152568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,6144,0.10495039621988933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,3584,0.0076117331782976794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,5120,0.08858880201975504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,3072,0.007034666836261749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,4096,0.06989866892496745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,2560,0.006234666705131531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,3584,0.058606934547424314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,2048,0.005502933263778686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,3072,0.04806293249130249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,51200,0.7937130610148112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,2560,0.03847786585489909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,1536,0.004956800242265066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,2048,0.03169813354810079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,1024,0.004293333490689596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,64,5120,0.0125109334786733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,1536,0.026682666937510174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,768,0.003903999924659729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,1024,0.022881066799163817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,512,0.0035487999518712364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,256,0.0032405334214369455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,768,0.02020053267478943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,65536,1.0045984268188477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,128,0.0029535998900731405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,512,0.01824000080426534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,64,0.002845866729815801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,256,0.016312533617019655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3841,32,32,0.002885333448648453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,128,0.01539413332939148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,64,2048,0.03198933402697245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,32,0.014776532848676046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3841,32,64,0.015346133708953857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,16384,5.764515177408854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,16384,7.1453099568684895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,12288,4.412944030761719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,12288,8.444244384765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,16384,10.933387247721354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,10240,3.8834465026855467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,10240,6.095524088541667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,10240,8.036461893717448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,12288,5.449585978190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,8192,3.0274729410807293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,8192,6.136359659830729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,7168,2.6146954854329425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,7168,3.7478004455566407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,51200,20.04342041015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,7168,4.919532775878906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,6144,2.2467051188151044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,8192,3.8478772481282553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,6144,4.15506337483724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,5120,1.8695189158121746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,5120,2.7656618754069013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,4096,1.5116447448730468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,5120,3.630889638264974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,4096,1.8539071400960285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,4096,2.8865941365559893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,3584,1.4022293090820312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,51200,22.867185465494792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,3584,1.6739434560139972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,3072,1.2235307057698568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,3584,2.6099306742350263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,3072,2.202525838216146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,2560,1.0407456080118815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,2560,1.2806442260742188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,6144,2.875446319580078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,2048,1.460589853922526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,2048,0.8548959732055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,2560,1.9937044779459636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,2048,1.0760842641194661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,1536,0.594220797220866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,1024,0.4642335891723633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,1536,1.1595413208007812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,1536,0.853650156656901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,1024,0.9449600219726563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,768,0.5781930923461914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,768,0.40366506576538086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,512,0.4005621274312337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,512,0.35719359715779625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,768,0.6117120107014974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,256,0.2725386619567871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,256,0.2226037343343099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,128,0.21571946144104004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,128,0.21405332883199057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,128,0.37188800175984704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,64,0.1967349370320638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,3072,1.4870474497477213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,64,0.20930773417154946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,32,0.19203306833902994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,65536,32,0.20238186518351237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,1024,0.5851125081380208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,512,0.4355850537618001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,65536,256,0.36701440811157227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,65536,51200,35.7772216796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,16384,8.670484415690105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,16384,4.965615844726562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,51200,14.78036092122396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,12288,6.776726277669271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,65536,19.946586100260415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,16384,5.617468770345052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,51200,19.021968587239584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,12288,3.976678466796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,12288,5.077820841471354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,10240,3.1093599955240885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,10240,5.664372253417969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,8192,2.4277684529622396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,8192,3.002001190185547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,8192,4.438443501790365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,51200,28.67887776692708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,10240,3.5868052164713538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,7168,1.9413888295491535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,65536,29.908854166666668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,7168,4.075718434651693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,7168,2.6919509887695314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,6144,3.4087860107421877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,6144,1.8531808217366534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,6144,2.206591033935547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,5120,2.816350809733073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,5120,1.4125162760416665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,4096,1.1995285034179688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,5120,1.7634143829345703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,3584,1.0045855840047202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,4096,2.2069493611653646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,3584,1.8853941599527997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,3072,0.8693909327189127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,3072,1.6145226796468097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,2560,0.7740714391072591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,3072,1.0534346898396811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,2560,1.3557664235432942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,65536,35.96751708984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,2048,1.1439189910888672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,4096,1.4863316853841146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,1536,0.8229557037353515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,2048,0.6052725474039714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,2048,0.7699136098225912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,1024,0.5509557088216146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,1024,0.38110081354777015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,1024,0.5153738657633464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,768,0.44835519790649414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,768,0.30623038609822595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,512,0.31723947525024415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,512,0.24001065889994302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,3584,1.2616288503011068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,768,0.4582389195760091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,256,0.19612693786621094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,256,0.19092373847961425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,128,0.17158719698588054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,512,0.35723625818888344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,128,0.17581653594970703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,256,0.3497685432434082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,64,0.16039466857910156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,128,0.2910143852233887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,64,0.17096106211344403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,51200,32,0.15726505915323893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,32,0.16640000343322753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,51200,1536,0.4797919909159343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,2560,0.9053120295206705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,51200,1536,0.6113471984863281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,16384,2.8612607320149737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,16384,1.7211989084879558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,51200,5.230383809407552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,12288,2.0907819112141928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,51200,5.644879150390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,65536,6.705806986490886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,12288,1.2712788899739584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,12288,1.3943359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,10240,1.026042683919271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,10240,1.7553204854329425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,10240,1.260268783569336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,16384,1.8816032409667969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,51200,8.731317138671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,8192,1.3993364969889321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,8192,0.8971925099690756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,7168,0.7878986358642578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,7168,1.3027754465738932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,6144,0.6684725443522136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,6144,1.0688191731770833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,6144,0.7435850779215495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,5120,0.5719914754231771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,5120,0.892634646097819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,4096,0.7094101587931315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,4096,0.4508639971415202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,65536,8.134312438964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,3584,0.4314293225606282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,65536,12.228514607747396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,3584,0.6461962381998698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,3072,0.630020268758138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,3072,0.3726261456807455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,3072,0.3352191925048828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,8192,0.9063210805257162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,2560,0.2752415974934896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,2560,0.508354123433431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,2048,0.22975467046101888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,2048,0.3829930623372396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,7168,0.7730762481689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,1536,0.2598090648651123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,1536,0.20165653228759767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,1024,0.18622934023539225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,1024,0.14688746134440106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,1536,0.24760212898254394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,768,0.14806933403015138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,768,0.12235946655273437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,512,0.10716160138448079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,5120,0.5662325541178386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,512,0.08921813170115153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,256,0.064955735206604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,256,0.07447360356648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,4096,0.4740960121154785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,128,0.057150932153066006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,128,0.06933333079020182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,128,0.10343893369038899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,3584,0.4080469449361165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,64,0.06864426930745443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,2560,0.3045728047688802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,32,0.05288000106811523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,16384,32,0.06850346724192301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,2048,0.2588160037994385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,16384,64,0.05235840082168579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,1024,0.16575466791788737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,768,0.14732267061869303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,512,0.12203199863433838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,16384,256,0.10096639792124432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,16384,2.1785909016927083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,16384,1.242804209391276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,51200,3.922896067301432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,16384,1.3622314453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,65536,5.0287938435872395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,12288,1.6248767852783204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,51200,4.523284403483073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,12288,0.9524192174275716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,65536,6.050172932942709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,10240,1.291330083211263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,10240,0.7892800013224284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,10240,0.8688959757486978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,51200,6.428170776367187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,8192,0.6570058822631836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,8192,0.6767509460449219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,7168,0.593829345703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,7168,0.8960863749186198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,7168,0.5855648040771484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,6144,0.5150698661804199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,6144,0.7719562530517579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,5120,0.42695147196451827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,65536,8.318698628743489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,5120,0.6750090916951497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,8192,1.0596832275390624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,4096,0.5580479939778645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,4096,0.339682133992513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,4096,0.3610602696736654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,3584,0.2858463923136393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,3584,0.44777386983235673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,3072,0.3743189175923666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,3072,0.2577973365783691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,3072,0.2666389306386312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,2560,0.3445472081502279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,2560,0.22870933214823402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,12288,1.002016003926595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,2048,0.2556373278299967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,2048,0.20043946901957194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,1536,0.23435734113057455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,1536,0.15091946919759114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,1536,0.19013546307881674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,1024,0.11405333677927654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,1024,0.16626027425130208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,1024,0.14852479298909504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,768,0.11218986511230469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,768,0.10405226548512776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,512,0.08105599880218506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,512,0.07741546630859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,768,0.1321760018666585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,256,0.05493333339691162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,512,0.09405439694722494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,256,0.060437333583831784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,128,0.04477546612421672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,256,0.08147626717885335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,128,0.05475946664810181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,64,0.03972586790720622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,128,0.07194773356119791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,12288,32,0.040591998895009355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,64,0.05523519913355509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,12288,32,0.05489173332850138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,6144,0.5000266710917155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,5120,0.42528425852457685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,3584,0.3113055864969889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,2560,0.23158720334370933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,12288,2048,0.19496533075968425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,16384,1.0789258321126303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,16384,1.8482186635335285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,65536,4.42754872639974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,16384,1.1974037170410157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,51200,3.507035827636719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,12288,1.3493813832600912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,12288,0.8757866541544596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,10240,1.1742677052815755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,10240,0.6990901311238606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,51200,5.525983174641928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,8192,0.5712426503499348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,8192,0.9494431813557943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,65536,7.048907979329426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,8192,0.5865695953369141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,7168,0.8594933191935221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,7168,0.5408341089884441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,7168,0.5185471852620442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,6144,0.6795434951782227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,6144,0.4505695978800456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,5120,0.565564791361491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,5120,0.3943690617879232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,65536,5.046443684895833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,5120,0.4023882548014323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,4096,0.31093759536743165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,12288,0.8271423975626627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,51200,3.9467445373535157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,4096,0.46529280344645185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,3584,0.42134720484415694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,3584,0.28472747802734377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,3584,0.3157151858011881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,3072,0.39187733332316077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,3072,0.2598464012145996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,3072,0.24367146492004393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,2560,0.306219736735026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,2560,0.2246837298075358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,6144,0.47281494140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,2560,0.2233941396077474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,2048,0.23031360308329263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,10240,0.709654426574707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,2048,0.18348479270935059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,1536,0.12108053366343181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,1536,0.17281920115152996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,1536,0.15217280387878418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,1024,0.10786773363749187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,1024,0.09280746777852376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,1024,0.12057387034098307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,768,0.08484906355539958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,768,0.07877546946207682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,512,0.06440213521321615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,768,0.09613760312398276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,512,0.07011093298594157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,512,0.08070507049560546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,256,0.051734399795532224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,256,0.06688319842020671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,128,0.05142506758371988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,128,0.04846506516138713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,64,0.034790400664011636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,128,0.06581653356552124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,10240,32,0.03535466591517131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,64,0.048853333791097006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,10240,32,0.048365867137908934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,256,0.06966719627380372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,4096,0.296395746866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,10240,2048,0.17517760594685872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,16384,1.4230730692545572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,16384,0.9366549173990885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,65536,3.753193664550781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,65536,4.000712585449219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,16384,0.9890975952148438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,51200,2.915152994791667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,51200,4.346190897623698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,12288,0.7071903864542644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,12288,1.087285359700521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,51200,3.072133382161458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,10240,0.5841119766235352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,10240,0.5889759699503581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,10240,0.9203829447428384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,8192,0.7181941350301106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,8192,0.5110389391581218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,7168,0.420305061340332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,7168,0.6155466715494792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,7168,0.4634592056274414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,6144,0.38554560343424477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,6144,0.5507424036661784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,65536,6.281075032552083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,6144,0.36038185755411783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,5120,0.46103680928548174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,5120,0.32987521489461263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,4096,0.3628320058186849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,12288,0.7706581115722656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,4096,0.26077226003011067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,3584,0.33412691752115886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,3584,0.2499114672342936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,3072,0.19313920338948568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,3072,0.2110208034515381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,3072,0.25467839241027834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,2560,0.22673494021097818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,2560,0.16143147150675458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,2560,0.17775786717732747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,2048,0.17541227340698243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,2048,0.15166932741800945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,1536,0.13708906173706054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,1536,0.11242773532867431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,1024,0.09869653383890788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,1024,0.09725866317749024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,768,0.07631999651590983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,768,0.07297706604003906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,768,0.08125332991282144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,3584,0.22979839642842612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,512,0.05699199835459391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,512,0.05875840187072754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,512,0.06647253433863322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,256,0.03859413464864095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,256,0.04661013285319011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,128,0.031591467062632245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,128,0.04169706503550212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,128,0.050001064936319985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,64,0.02840106685956319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,8192,0.4559040069580078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,8192,32,0.02953280011812846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,64,0.0423253337542216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,1024,0.0851733366648356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,8192,32,0.04443946679433187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,4096,0.24050453503926597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,5120,0.30196479161580403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,2048,0.13907306989034016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,1536,0.1136629343032837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,8192,256,0.053653331597646084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,65536,3.3752896626790365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,51200,2.681995646158854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,65536,3.4773450215657555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,16384,1.2482869466145834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,51200,3.80976308186849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,16384,0.84541867574056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,12288,0.6482880274454753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,16384,0.9047733306884765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,12288,0.9275573094685873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,65536,4.860869344075521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,10240,0.5399210611979167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,12288,0.6081696192423502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,10240,0.5898176193237304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,8192,0.6220511754353841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,10240,0.7906623840332031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,8192,0.4484117190043132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,7168,0.38003412882486975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,7168,0.42116266886393233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,7168,0.5417727788289388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,6144,0.4887071927388509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,6144,0.33619734446207683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,5120,0.26208747227986656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,5120,0.28853867848714193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,5120,0.3929792086283366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,4096,0.32948586146036785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,4096,0.23362879753112792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,3584,0.2589941342671712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,3584,0.21892053286234536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,3584,0.2213301340738932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,3072,0.22455360094706217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,3072,0.181330140431722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,2560,0.20462400118509927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,2560,0.1560991923014323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,2560,0.1507285277048747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,2048,0.13051520188649496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,1536,0.11872213681538898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,1536,0.10151039759318034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,1536,0.10447893142700196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,51200,2.7282869974772135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,1024,0.08389653364817301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,1024,0.08178239663441976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,1024,0.09493866761525473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,768,0.0665226658185323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,768,0.08024426301320395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,512,0.05532480080922445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,512,0.05892159938812256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,2048,0.16219733556111654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,256,0.033505066235860186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,256,0.04346666733423869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,256,0.048903465270996094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,128,0.02834133307139079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,8192,0.4071498552958171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,128,0.03901120026906331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,64,0.024977066119511924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,128,0.044964265823364255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,7168,32,0.027221333980560303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,64,0.04040426810582479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,7168,32,0.04227840105692546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,6144,0.3089994748433431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,4096,0.21436266899108886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,3072,0.1731690724690755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,2048,0.1242677370707194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,768,0.07142613728841146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,7168,512,0.058645331859588624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,16384,0.9888287862141928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,65536,2.9161387125651044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,16384,0.7264821370442708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,65536,3.0878890991210937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,51200,2.3562347412109377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,12288,0.5645514806111653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,51200,3.3725824991861977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,12288,0.7420192082722982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,12288,0.5884970982869466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,10240,0.6422047932942708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,65536,4.1663818359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,10240,0.47259092330932617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,8192,0.5361482620239257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,8192,0.4090239842732747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,7168,0.33707094192504883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,7168,0.45398718516031905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,7168,0.33911040623982747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,6144,0.3709952036539713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,6144,0.2936298688252767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,5120,0.2589695930480957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,5120,0.3254538536071777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,4096,0.24750612576802572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,4096,0.19406506220499675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,10240,0.45488640467325847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,3584,0.2177130699157715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,3584,0.17701120376586915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,3072,0.18906133969624836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,3072,0.15452799797058106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,51200,2.358662414550781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,2560,0.19785599708557128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,4096,0.22303679784139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,16384,0.7169823964436849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,2560,0.16002666155497233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,2048,0.14123199780782064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,2048,0.1348746617635091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,8192,0.3621098518371582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,1536,0.10165546735127766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,1536,0.09496959845225016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,1536,0.09409493605295817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,6144,0.2685002644856771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,1024,0.07335999806722006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,1024,0.06890880266825358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,768,0.05672533512115478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,5120,0.23019092877705893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,768,0.05890239874521891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,768,0.060725335280100504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,512,0.04284480015436808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,512,0.04963200092315674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,512,0.05054293473561605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,256,0.029487999280293782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,256,0.03981546560923259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,256,0.04192426602045695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,128,0.02413653333981832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,128,0.03707306782404582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,128,0.038661332925160725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,64,0.022344533602396646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,6144,32,0.023202133178710938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,64,0.036298668384552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,6144,32,0.03645546833674113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,3584,0.1657109260559082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,3072,0.14571626981099445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,2560,0.12649919986724853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,2048,0.10733333428700764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,6144,1024,0.06994453271230062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,16384,0.8754218419392904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,65536,2.6657173156738283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,16384,0.6828671773274739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,12288,0.6494442621866863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,65536,3.7711062113444007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,51200,2.9466580708821613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,12288,0.5288714726765951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,10240,0.5460831960042317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,10240,0.44145708084106444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,8192,0.3543445269266764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,8192,0.46234451929728193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,51200,2.091566975911458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,7168,0.40184532801310224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,7168,0.30868800481160485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,6144,0.31845439275105797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,6144,0.2765418688456217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,12288,0.5451893488566081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,5120,0.2734389305114746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,5120,0.2343520005544027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,65536,3.1001462300618488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,16384,0.7123221079508464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,4096,0.2442549387613932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,4096,0.20801493326822915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,3584,0.19918400446573895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,3584,0.17400426864624025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,10240,0.48041388193766277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,6144,0.27192214330037434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,3072,0.17222506205240887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,3072,0.14402666091918945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,2560,0.13971625963846843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,2560,0.12080106735229493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,51200,2.373565928141276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,2048,0.13664533297220866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,2048,0.12182826995849609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,7168,0.32739413579305016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,1536,0.08863999843597412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,1536,0.08416213194529215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,8192,0.3664725303649902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,1024,0.06579946676890056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,1024,0.06653120120366415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,3072,0.1413973331451416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,768,0.05162133375803629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,768,0.055940266450246176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,4096,0.18848427136739093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,512,0.03924479881922404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,512,0.0469322681427002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,1536,0.07708693345387777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,256,0.027137066920598345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,256,0.036848000685373944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,2560,0.12604479789733886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,128,0.022321067253748574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,128,0.0333568016688029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,5120,0.23269012769063316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,64,0.01882773240407308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,3584,0.16446186701456705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,64,0.033684265613555905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,5120,32,0.019335466623306274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,5120,32,0.03374720017115275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,1024,0.06165759960810343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,2048,0.1024074633916219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,65536,2.2349782307942707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,65536,2.7328458150227863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,51200,2.2139872233072917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,16384,0.6755541483561198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,51200,1.7560352325439452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,16384,0.5959231694539388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,256,0.03234453399976094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,128,0.027287467320760088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,12288,0.5034677187601726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,12288,0.430022398630778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,10240,0.36118507385253906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,512,0.039061331748962404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,10240,0.42370026906331376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,8192,0.35797653198242185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,5120,768,0.049218134085337324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,8192,0.3219125429789225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,7168,0.28988161087036135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,7168,0.27540906270345056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,6144,0.24668587048848473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,6144,0.23212052981058756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,65536,2.464343516031901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,12288,0.4604448000590007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,5120,0.22264533042907714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,5120,0.19344533284505208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,4096,0.16658026377360027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,4096,0.15778026580810547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,16384,0.6009471893310547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,3584,0.1547818660736084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,3584,0.15012267430623372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,10240,0.4006442705790202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,3072,0.12682560284932454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,3072,0.12686080137888592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,7168,0.2719680150349935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,2560,0.10900906721750896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,51200,2.0398272196451823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,2560,0.10586667060852051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,2048,0.08930133183797201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,8192,0.31223573684692385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,2048,0.08845973014831543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,1536,0.06935786406199138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,1536,0.0723861296971639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,6144,0.2318112055460612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,1024,0.04966080188751221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,1024,0.05683626731236776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,4096,0.1577184041341146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,768,0.04025813341140747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,768,0.048634668191274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,3072,0.11853653589884441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,512,0.03136000037193298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,512,0.041010133425394696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,5120,0.19597652753194172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,256,0.02216106653213501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,256,0.03261973261833191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,1536,0.06477440198262532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,128,0.017248000701268515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,128,0.029627732435862225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,3584,0.14050025939941407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,64,0.014363732933998109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,64,0.02943893273671468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,4096,32,0.015275733669598899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,4096,32,0.02885546684265137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,2560,0.10477546850840251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,1024,0.05222506523132324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,2048,0.08728319803873698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,65536,2.5451210021972654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,65536,2.0644415537516276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,51200,2.0108693440755205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,51200,1.6246634165445966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,16384,0.6078528086344401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,256,0.027796266476313274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,128,0.02315839926401774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,16384,0.534117317199707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,512,0.03317013382911682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,12288,0.42841386795043945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,10240,0.35227413177490235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,12288,0.40427627563476565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,10240,0.3511754671732584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,8192,0.2887114524841309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,8192,0.2822208086649577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,4096,768,0.04183040062586467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,7168,0.2537386735280355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,7168,0.24326186180114745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,6144,0.21623360315958656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,12288,0.39476372400919596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,6144,0.2166752020517985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,5120,0.1820842742919922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,65536,2.177604166666667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,5120,0.18259305953979493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,4096,0.1477205276489258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,51200,1.757101821899414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,4096,0.14910826683044434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,3584,0.13045653502146404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,3584,0.1309386650721232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,3584,0.12338879903157551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,3072,0.11316800117492676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,3072,0.11399253209431966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,16384,0.5297941207885742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,2560,0.09879999955495199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,2560,0.1065941333770752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,10240,0.3535242716471354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,2048,0.07830933729807535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,2048,0.08653333187103271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,8192,0.26479786237080893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,2048,0.08042986392974853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,1536,0.06113066673278809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,1536,0.07015039920806884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,1024,0.04568746487299601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,1024,0.05514986515045166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,7168,0.2333984057108561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,768,0.03577173153559367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,768,0.04572480122248332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,6144,0.19703253110249835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,5120,0.1708906650543213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,512,0.027937066555023194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,512,0.03834240039189656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,4096,0.13630612691243488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,256,0.019876267512639365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,256,0.03120853304862976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,3072,0.10452373027801513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,128,0.015384533007939658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,128,0.028225066264470418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,2560,0.09316159884134928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,64,0.013674666484196981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,64,0.027730133136113482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3584,32,0.01415786643822988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3584,32,0.02720959981282552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,1536,0.058413867155710855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,1024,0.04700800180435181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,768,0.038165334860483804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,512,0.030584534009297688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,256,0.024868265787760416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,65536,1.9235370635986329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,65536,2.2619733174641925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,16384,0.4875957489013672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,51200,1.6998485565185546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,51200,1.5099263509114584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,16384,0.49645760854085286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,12288,0.390777587890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3584,128,0.020616533358891805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,12288,0.39768959681193033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,10240,0.31468159357706704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,12288,0.3619733174641927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,10240,0.33431145350138347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,8192,0.25284053484598795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,8192,0.2685461362202962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,7168,0.22275840441385902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,7168,0.2357685407002767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,6144,0.19253759384155272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,6144,0.2035136063893636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,5120,0.16108373006184895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,5120,0.17159040768941242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,65536,1.9914826711018878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,4096,0.14938027064005535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,4096,0.15728747049967448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,16384,0.4786677360534668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,3584,0.1200544039408366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,3584,0.1287893295288086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,10240,0.31177921295166017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,51200,1.5631605784098306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,3072,0.09990932941436767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,3072,0.12565226554870607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,8192,0.24993599255879723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,2560,0.09259733359018961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,2560,0.09337600072224936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,2048,0.06890986760457357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,7168,0.21673812866210937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,6144,0.1826346715291341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,2048,0.07976319789886474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,2048,0.06857813199361165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,1536,0.05412373145421347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,1536,0.06558506488800049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,1024,0.04017066558202108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,1024,0.050163201491038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,768,0.03260266582171122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,768,0.04270293315251668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,768,0.034687999884287515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,512,0.02521066665649414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,5120,0.15377066930135092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,512,0.03574719826380412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,256,0.017638399203618368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,256,0.03013226588567098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,4096,0.1274837334950765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,128,0.013289599617322286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,128,0.025851732492446898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,128,0.01838399966557821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,64,0.011892267068227132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,3584,0.11384212970733643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,64,0.02610986630121867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,3072,32,0.012852266430854797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,3072,32,0.026016000906626386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,2560,0.0851861317952474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,1536,0.052085332075754796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,1024,0.0420576016108195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,512,0.02691733241081238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,256,0.022156800826390585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,3072,3072,0.09541866779327393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,65536,1.8787221272786456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,16384,0.40779946645100906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,51200,1.3758421579996745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,16384,0.44947519302368166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,51200,1.3577045440673827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,12288,0.3069237391153971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,10240,0.255564800898234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,12288,0.3771679878234863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,10240,0.29785172144571936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,8192,0.2118325392405192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,8192,0.23894186019897462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,7168,0.18150399525960287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,65536,1.7391882578531903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,7168,0.20969600677490235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,7168,0.19567039807637532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,6144,0.156496000289917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,6144,0.18089706103007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,12288,0.3289152145385742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,6144,0.16556053161621093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,5120,0.13219626744588214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,5120,0.15301012992858887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,5120,0.1413632074991862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,4096,0.10646293163299561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,4096,0.12661333084106446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,3584,0.0949461301167806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,3584,0.11200213432312012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,65536,1.8195893605550129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,3072,0.09439893563588461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,16384,0.44087467193603513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,3072,0.1093600034713745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,51200,1.3849620819091797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,2560,0.08059626420338949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,2560,0.0932533343633016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,2560,0.08144213358561198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,2048,0.06762879689534505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,2048,0.07523946762084961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,8192,0.22404266993204752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,1536,0.0463648001352946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,1536,0.05721386671066284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,1024,0.03237546682357788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,1024,0.0430730660756429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,10240,0.28165012995402017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,768,0.026226133108139038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,768,0.03683946530024211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,768,0.030267733335494994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,512,0.02039253314336141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,512,0.03141760031382243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,3584,0.101910400390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,256,0.013874133427937826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,256,0.02542399962743123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,128,0.011632000406583149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,128,0.02396906614303589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,3072,0.08735786279042562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,64,0.01088213324546814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,64,0.024074665705362954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,4096,0.11589759985605877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2560,32,0.011255466938018798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2560,32,0.023935999472935995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,2048,0.06238186756769816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,1536,0.047068798542022706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,1024,0.03696746826171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,51200,1.105135981241862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,65536,1.4547743479410806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,65536,1.56692263285319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,51200,1.2300000508626303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,16384,0.3751797358194987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,512,0.024293333292007446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,12288,0.276914119720459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,16384,0.40558614730834963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,16384,0.31306772232055663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,256,0.018999467293421425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,12288,0.30888001124064124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2560,128,0.01591679950555166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,51200,1.04180056254069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,12288,0.2678208033243815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,10240,0.19850026766459147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,10240,0.2625162601470947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,8192,0.16251734097798665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,7168,0.1538464069366455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,6144,0.12335573037465412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,7168,0.14584213892618816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,8192,0.21066880226135254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,7168,0.18610986073811847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,5120,0.10581653118133545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,6144,0.12433173656463622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,5120,0.10505066712697346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,6144,0.1724064032236735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,5120,0.13737386067708332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,4096,0.08595840136210123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,3584,0.07762773036956787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,4096,0.0862272024154663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,3584,0.0997717301050822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,3072,0.06690239906311035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,3584,0.07772479852040609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,3072,0.08798826535542806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,3072,0.06703466574350993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,2560,0.05768746534983317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,2560,0.0587007999420166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,2560,0.0766869306564331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,2048,0.04543253183364868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,2048,0.06519466638565063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,1536,0.036160000165303546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,1536,0.05007786750793457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,1024,0.0369429349899292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,1024,0.03206506570180257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,768,0.020039467016855876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,768,0.032364799578984575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,768,0.027818665901819868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,10240,0.20447039604187012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,512,0.015305599570274353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,512,0.02829119960467021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,512,0.022836265961329143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,256,0.011427199840545655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,256,0.024305067459742227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,256,0.0185589333375295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,128,0.009913600484530131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,128,0.022410666942596434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,128,0.01665173371632894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,64,0.009403733412424724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,64,0.022590933243433635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,4096,0.11439146995544433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,32,0.009783466657002766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,65536,1.3015552520751954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,2048,32,0.023033599058787026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,2048,1024,0.02469546596209208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,65536,1.0299370447794596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,51200,0.8040565490722656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,8192,0.16386987368265787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,65536,1.4266539255777995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,16384,0.251417605082194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,16384,0.3674869219462077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,2048,0.049960533777872726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,51200,1.1234432220458985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,12288,0.18950400352478028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,12288,0.27750720977783205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,10240,0.23320107460021972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,8192,0.12642133235931396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,2048,1536,0.04210986693700154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,8192,0.19050025939941406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,10240,0.15907519658406574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,7168,0.17044587135314943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,7168,0.111572265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,6144,0.09610559940338134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,6144,0.14749226570129395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,12288,0.22494293848673502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,5120,0.08136320114135742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,5120,0.12651413281758625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,16384,0.2916927973429362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,4096,0.06643093427022298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,4096,0.10323200225830079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,10240,0.1901408036549886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,3584,0.05822933514912924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,3584,0.09174079895019531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,51200,0.924186642964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,3072,0.05479466517766317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,65536,1.1736148834228515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,3072,0.08489920298258463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,2560,0.04655253489812215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,2560,0.07224533557891846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,6144,0.11745920181274414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,2048,0.03484479983647664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,2048,0.05575999816258749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,7168,0.13493547439575196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,1536,0.0274399995803833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,1536,0.042872532208760576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,8192,0.15112853050231934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,1024,0.02062506675720215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,1024,0.03295466701189677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,3072,0.05932799975077311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,768,0.016821332772572837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,768,0.028999465703964233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,4096,0.07930560111999511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,512,0.012873599926630655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,512,0.024949334065119424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,1536,0.03359253406524658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,256,0.009329066673914591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,256,0.024171733856201173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,5120,0.0964202642440796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,128,0.007392000158627827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,128,0.02190293272336324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,2560,0.051667201519012454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,64,0.008601599931716919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,64,0.021811199188232423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1536,32,0.009070932865142822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1536,32,0.02226026654243469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,3584,0.06791679859161377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,65536,0.6496938705444336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,2048,0.04391359885533651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,65536,1.2374795277913413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,1024,0.025808000564575197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,51200,0.5016480127970377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,16384,0.1638271967569987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,16384,0.32210346857706706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,16384,0.21934186617533363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,51200,0.9655509312947592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,12288,0.12277546723683674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,12288,0.16356587409973145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,12288,0.2438432057698568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,10240,0.10339199701944987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,10240,0.20587306022644042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,10240,0.13714453379313152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,8192,0.08380266825358072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,8192,0.1718229293823242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,8192,0.10812053680419922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,7168,0.07356800238291422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,7168,0.15075519879659016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,6144,0.06385386784871419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,6144,0.1320863962173462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,6144,0.0840458631515503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,5120,0.053998935222625735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,128,0.011575466394424439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,256,0.013783466815948487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,5120,0.11253653367360432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,5120,0.0706666628519694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,4096,0.04412586688995361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,4096,0.09213653405507406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,65536,0.8168415705362955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,3584,0.039000535011291505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,3584,0.08284479777018229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,3072,0.035207466284434004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,3072,0.07062186400095621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,512,0.01702186663945516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,2560,0.028769065936406452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,7168,0.09863359928131103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,2560,0.041000533103942874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,2560,0.057791999975840246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,2048,0.024127999941507973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,2048,0.04660160144170125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,2048,0.03449600140253703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,1536,0.01935466726620992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,1536,0.03671893278757731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,1024,0.01427733302116394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,1024,0.029798400402069092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,1024,0.021270400285720824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,51200,0.6376757303873698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,768,0.01195199986298879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,768,0.02579200069109599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,512,0.00986346701780955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,512,0.02298133373260498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,512,0.015481600165367126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,256,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,256,0.019307732582092285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1536,768,0.021687465906143188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,128,0.006078933179378509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,128,0.018359466393788656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,1536,0.02770666678746541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,64,0.0052490666508674625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,64,0.01809920072555542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,1024,32,0.005452799797058106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,1024,32,0.017870932817459106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,65536,0.5102933247884114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,4096,0.05814400116602579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,3584,0.053201067447662356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,51200,0.39665600458780925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,3072,0.04704426527023316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,16384,0.12840853532155355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,51200,0.9147498448689779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,65536,1.1866794586181642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,16384,0.30294507344563804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,12288,0.09772693316141765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,768,0.018832000096638997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,12288,0.23099946975708008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,10240,0.08088320096333822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,10240,0.12793920040130616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,10240,0.19451840718587238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,8192,0.06538666486740112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,8192,0.1598954677581787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,7168,0.05853973229726156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,128,0.011083733042081196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,7168,0.14262933731079103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,6144,0.05055466492970785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,6144,0.12431999842325847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,1024,256,0.012497066458066305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,5120,0.04322559833526611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,5120,0.10600106716156006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,65536,0.7660970687866211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,4096,0.03472426732381185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,51200,0.6091786702473958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,4096,0.08651626904805501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,3584,0.03051946759223938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,16384,0.1977333386739095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,3584,0.04914453426996867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,3584,0.07508052984873453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,3072,0.02645333409309387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,3072,0.06392319997151694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,3072,0.042916266123453777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,2560,0.022589866320292154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,2560,0.05262826681137085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,2048,0.01879040002822876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,2048,0.04240640004475911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,8192,0.10404053529103596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,1536,0.015042133132616677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,2048,0.030534400542577105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,1536,0.03361066579818726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,7168,0.09118293126424154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,1024,0.0276309331258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,6144,0.07882239818572997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,768,0.00995733340581258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,768,0.02483839988708496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,12288,0.15359466870625813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,512,0.008020266890525818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,512,0.021226666371027627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,5120,0.06827092965443929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,256,0.006366933385531108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,1024,0.01134933332602183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,256,0.018568533658981323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,128,0.005350400010744731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,128,0.018093866109848023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,4096,0.05500799814860026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,64,0.004693333307902018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,64,0.017866667111714682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,768,32,0.004676266511281332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,768,32,0.01766293247540792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,2560,0.03811093171437581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,65536,0.3462453206380208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,1536,0.024158932765324912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,1024,0.019106133778889974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,65536,0.6161802927652995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,51200,0.26779521306355797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,768,0.01625279982884725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,65536,1.0881375630696615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,16384,0.0877791961034139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,512,0.013715199629465737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,51200,0.48751360575358077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,16384,0.16491093635559081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,12288,0.0664576013882955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,16384,0.28157227834065757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,10240,0.05585600137710571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,12288,0.2154794692993164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,10240,0.18219520250956217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,51200,0.8516661326090494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,8192,0.04468799829483032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,8192,0.08429226875305176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,7168,0.039686401685078934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,8192,0.14948053359985353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,256,0.010545066992441813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,7168,0.07447146574656169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,6144,0.0348416010538737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,7168,0.1331818660100301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,768,128,0.009231999516487122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,5120,0.02976106603940328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,4096,0.02361066738764445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,6144,0.11533013184865315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,5120,0.09843839804331461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,3584,0.021052799622217813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,4096,0.08019733428955078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,3584,0.04112533330917358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,3584,0.06882133483886718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,3072,0.018683733542760213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,3072,0.05873173475265503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,3072,0.03611840009689331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,2560,0.016192000110944113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,2560,0.04757546583811442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,2048,0.013821867108345032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,2048,0.038321065902709964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,2048,0.025598933299382527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,1536,0.011142399907112122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,1536,0.0310208002726237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,1024,0.008781866232554118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,1024,0.025335466861724852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,1024,0.015997866789499916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,768,0.007685333490371704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,768,0.022949333985646567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,768,0.014103466272354126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,512,0.006400000055631001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,512,0.020035199324289956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,512,0.011853866775830587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,256,0.005165866514046987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,256,0.017997866868972777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,256,0.00936853289604187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,128,0.004458666841189066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,128,0.017256534099578856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,128,0.0086218665043513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,64,0.004380799829959869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,64,0.01655466655890147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,512,32,0.00443200021982193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,12288,0.12400960127512614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,10240,0.10382613341013591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,65536,0.20980799992879234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,6144,0.06444053252538046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,4096,0.044814932346344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,51200,0.1655669371287028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,2560,0.03059626619021098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,65536,1.0282943725585938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,16384,0.0570741335550944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,16384,0.26925760904947915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,512,32,0.016362667083740234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,12288,0.04475733439127604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,51200,0.80469118754069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,10240,0.037573333581288657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,12288,0.11169919967651368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,1536,0.02036799987157186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,12288,0.20673492749532066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,10240,0.09342933495839437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,8192,0.03104746739069621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,10240,0.17384959856669108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,512,5120,0.05529173215230306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,7168,0.028116265932718914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,8192,0.07556479771931966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,8192,0.14257493019104003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,6144,0.02484053373336792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,6144,0.057335468133290615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,5120,0.021236266692479452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,7168,0.1256501356760661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,6144,0.10978773434956868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,4096,0.013267200191815695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,5120,0.09240852991739909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,4096,0.07418560187021891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,3584,0.012179199854532878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,3584,0.06326293150583903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,3072,0.01095360020796458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,3072,0.05281493266423544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,65536,0.5540629069010417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,2560,0.009875200192133586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,2560,0.04172799984614055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,51200,0.4377663930257161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,16384,0.14726187388102213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,2048,0.008283733328183492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,2048,0.033868801593780515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,7168,0.06632853349049886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,1536,0.007336533566315968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,1536,0.027502934137980144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,4096,0.03970986604690552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,5120,0.048555731773376465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,1024,0.005948799848556519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,1024,0.013771733641624451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,3584,0.03667626778284709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,1024,0.023430399099985757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,768,0.00526506652434667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,512,0.0045621335506439206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,768,0.012090667088826498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,768,0.021331199010213218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,512,0.018604799111684164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,3072,0.03173440098762512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,256,0.01718399922053019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,128,0.003587199995915095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,128,0.01588373382886251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,256,0.003946666667858759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,64,0.003453866640726725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,64,0.015745066603024802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,256,32,0.003492266684770584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,256,32,0.015467733144760132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,2560,0.026716800530751546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,65536,0.19405333201090497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,2048,0.021934932470321654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,51200,0.15367040634155274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,65536,1.0125429153442382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,1536,0.01794346570968628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,16384,0.054997332890828456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,51200,0.7965226491292318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,12288,0.03795200188954671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,16384,0.2619765281677246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,512,0.010085333387056987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,256,0.008124800026416778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,10240,0.03213546673456828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,12288,0.2016160011291504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,256,128,0.0070816000302632645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,8192,0.026279467344284057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,10240,0.1702720006306966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,7168,0.023756800095240276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,8192,0.13818346659342448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,7168,0.12285226980845135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,6144,0.02136533260345459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,6144,0.10694506963094075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,6144,0.05572160085042318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,51200,0.4237845420837402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,5120,0.013911466797192892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,5120,0.04711466630299886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,4096,0.010088533163070679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,5120,0.0893280029296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,4096,0.038422401746114096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,4096,0.07131840387980143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,3584,0.00909546713034312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,3584,0.06116373141606649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,3584,0.03471039930979411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,3072,0.008408533533414204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,3072,0.05023359855016073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,2560,0.007333333293596904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,2560,0.03920746644337972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,65536,0.5318527857462565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,2048,0.006444799900054932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,2048,0.03225066661834717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,16384,0.14219519297281902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,1536,0.005738666653633118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,1536,0.026845866441726686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,12288,0.10913706620534261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,1024,0.004997333387533823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,1024,0.02323733369509379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,1024,0.012702932953834534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,10240,0.0914901336034139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,768,0.004494933287302653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,8192,0.07400639851888022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,512,0.003950933367013932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,7168,0.06417706807454428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,512,0.019115734100341796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,256,0.003960533440113068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,256,0.01688106656074524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,3072,0.0297760009765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,128,0.0033834666013717652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,128,0.015787733594576518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,2560,0.025100799401601155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,64,0.0031615999837716425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,768,0.021797333161036173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,128,32,0.0031328000128269195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,32,0.015408000349998474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,65536,0.18939092953999836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,2048,0.02032853364944458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,51200,0.15163307189941405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,1536,0.016549332936604818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,16384,0.05106879870096842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,65536,1.0125834782918295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,128,64,0.01530239979426066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,12288,0.0401802659034729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,10240,0.033801599343617754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,768,0.01123199959595998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,12288,0.19971200625101726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,51200,0.7905120213826498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,512,0.009237333138783773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,8192,0.028248532613118486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,10240,0.16855360666910807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,7168,0.025603200991948443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,6144,0.020680532852808634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,256,0.007380266487598419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,5120,0.011730133493741354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,7168,0.12151573499043782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,4096,0.008745599786440532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,5120,0.08966399828592936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,6144,0.10592319965362548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,3584,0.009326933821042379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,4096,0.06976426442464193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,3072,0.008646399776140849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,3584,0.05993280013402304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,3072,0.049225600560506184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,2048,0.00628053347269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,2560,0.03908373514811198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,1536,0.005469866593678792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,2048,0.03175999919573466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,1024,0.004625066618124644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,768,0.004172799984614054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,1536,0.027015467484792072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,1024,0.022565333048502605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,512,0.003731200098991394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,768,0.021153066555658975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,256,0.0033344000577926634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,512,0.018182400862375894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,128,0.003173333406448364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,256,0.016850133736928306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,64,0.0030805334448814393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,128,0.015862400333086647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,32,0.0030303999781608583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,64,0.015330132842063905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,32,0.015161599715550741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3840,128,128,0.006324266890684764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,65536,0.18634239832560223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,16384,0.04875306685765584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,16384,0.2631114641825358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,12288,0.03810559908548991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,16384,0.26102827390034994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,12288,0.20136639277140297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,64,8192,0.13740800221761068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,10240,0.032680533329645794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,64,2560,0.0071839998165766404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,10240,0.16996800104777018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,7168,0.024459733565648397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,6144,0.022476800282796226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,8192,0.13664746284484863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,5120,0.010629333058993022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,7168,0.12119786739349366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,6144,0.10483840306599934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,4096,0.00839466651280721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,5120,0.08836906750996908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,3584,0.007778133451938629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,3072,0.0071722666422526045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,4096,0.06993280251820883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,3584,0.058455467224121094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,2560,0.006404266754786174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,3072,0.04828586578369141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,2048,0.005679999788602194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,2560,0.0386133352915446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,1536,0.005004799862702688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,2048,0.030844799677530926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,1024,0.004257066547870636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,51200,0.14809172948201496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,1536,0.02707200050354004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,1024,0.022545067469278972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,512,0.0035146666069825493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,256,0.0031445334355036415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,768,0.020278400182723998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,512,0.018206934134165444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,256,0.016883200407028197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,128,0.002880000074704488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,64,0.0027722666660944624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,128,0.015431466698646545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,32,0.002809600035349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,64,0.015245866775512696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,32,0.014857600132624308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,8192,0.02701866626739502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,51200,0.7887360254923503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3840,32,65536,1.0090848286946614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3840,32,768,0.0038890667259693147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,12288,4.234147135416666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,16384,5.889455159505209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,12288,8.138843790690105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,16384,10.61183573404948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,16384,7.683147684733074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,12288,5.657398478190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,10240,3.7293515523274743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,10240,6.710319010416667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,8192,3.383263905843099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,8192,5.649982706705729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,8192,4.387158203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,7168,2.8049461364746096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,7168,5.051191711425782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,10240,4.594859822591146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,51200,19.273396809895836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,6144,2.28328857421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,7168,3.392903391520182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,6144,4.086274210611979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,6144,2.7557706197102863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,5120,1.8858773549397785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,5120,3.4037567138671876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,5120,2.231181844075521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,4096,1.5567264556884766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,3584,1.2299636840820312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,4096,2.7786890665690107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,3584,1.5819488525390626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,3584,2.421670277913411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,3072,1.1672021230061849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,3072,2.1433685302734373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,3072,1.4450069427490235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,2560,0.9671701431274414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,2560,1.7888832092285156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,2048,0.8281728108723959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,2560,1.1858292897542317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,2048,1.388866170247396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,4096,1.870529047648112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,1536,0.655784543355306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,1536,0.8467039744059244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,1536,1.0384298960367837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,1024,0.738484255472819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,1024,0.4689770698547363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,768,0.3626741409301758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,768,0.5348320007324219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,1024,0.739186159769694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,512,0.4127392133076985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,768,0.5752864201863607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,512,0.26872212092081704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,256,0.272054386138916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,256,0.22876693407694498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,512,0.4880821228027344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,128,0.2131253401438395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,256,0.34893439610799154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,128,0.21099732716878256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,64,0.1966357390085856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,32,0.19040106137593588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,64,0.20443946520487466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,65536,32,0.19714667002360026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,2048,0.9425994873046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,128,0.32617279688517253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,65536,51200,34.6936767578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,65536,51200,31.73502197265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,16384,4.64351552327474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,16384,8.955858357747395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,51200,15.363959757486978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,16384,5.872276306152344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,65536,18.80997314453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,12288,3.4146377563476564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,12288,6.3910771687825525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,12288,4.683016459147135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,10240,2.8512171427408854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,10240,5.4538111368815105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,8192,2.266998291015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,8192,4.296159871419271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,51200,25.796205647786458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,51200,27.381998697916668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,10240,3.584272003173828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,7168,2.0071200052897136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,8192,2.990839385986328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,7168,4.148355102539062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,6144,1.7205460866292317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,6144,2.142645263671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,5120,2.8426719665527345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,5120,1.6021952311197918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,6144,3.6778719584147135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,5120,1.7991626739501954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,4096,1.203155263264974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,65536,32.905810546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,4096,2.254841613769531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,3584,1.0241088231404623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,4096,1.428113047281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,7168,2.3858388264973955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,3072,0.9350592295328776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,3584,1.9290091196695964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,3584,1.354196294148763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,3072,1.120459747314453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,3072,1.6531360626220704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,2560,0.7590709050496419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,2560,1.3665995279947916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,65536,34.5814697265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,2048,0.6507808049519856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,2560,0.9662794748942056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,2048,1.1248672485351563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,1536,0.49730345408121746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,1024,0.3679669380187988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,1536,0.8471861521402995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,1024,0.4841194788614909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,1536,0.6496341069539388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,1024,0.5532224019368489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,768,0.2981120109558105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,768,0.4766549428304036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,512,0.3423487981160482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,256,0.18501760164896647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,256,0.2247327963511149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,512,0.3495541254679362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,256,0.31922985712687174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,128,0.17202026049296062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,128,0.1696213404337565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,64,0.15574080149332684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,128,0.2663253307342529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,51200,32,0.1514560063680013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,64,0.16607893308003743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,32,0.16096320152282714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,51200,512,0.21731840769449867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,2048,0.7689215977986653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,51200,768,0.4047082583109538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,16384,1.5469866434733073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,16384,2.731927490234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,51200,4.730882263183593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,65536,6.04045664469401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,16384,1.9209332784016926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,12288,1.143773905436198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,12288,1.9413664499918621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,12288,1.5089407602945963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,10240,0.9520970662434897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,51200,8.473198954264323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,10240,1.705022939046224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,65536,9.55662333170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,10240,1.1678549448649087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,8192,0.7936618804931641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,8192,1.5203936258951822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,8192,0.971454938252767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,7168,1.1509194691975912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,7168,0.6994154612223308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,7168,0.8321215947469076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,6144,0.9784330368041992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,6144,0.5802805582682292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,5120,0.50404052734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,5120,0.5913344065348307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,6144,0.7414016087849935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,5120,0.8651039759318033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,4096,0.39162559509277345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,65536,10.742850748697917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,4096,0.6768949508666993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,4096,0.4747189203898112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,3584,0.378602663675944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,3584,0.5998922983805339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,3072,0.4910901387532552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,3072,0.3323871930440267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,2560,0.26837333043416345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,2560,0.4162698745727539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,51200,6.507924397786458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,2048,0.3495989481608073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,2048,0.21640213330586752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,2048,0.2634773254394531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,1536,0.27904319763183594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,1024,0.1684714635213216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,1024,0.12976213296254474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,1536,0.19560640652974445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,1536,0.24871253967285156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,1024,0.18470293680826824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,768,0.14299306869506836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,768,0.10762453079223633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,512,0.10431466897328694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,512,0.0854581356048584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,512,0.13154133160909015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,256,0.06296106576919555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,256,0.07087146441141765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,128,0.0552512009938558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,128,0.06436479886372884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,64,0.05097279946009318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,64,0.06444799900054932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,16384,32,0.05097599824269613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,16384,32,0.06375893354415893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,3072,0.36549011866251624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,3584,0.40729811986287434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,2560,0.294049072265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,768,0.140720001856486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,256,0.0964128017425537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,16384,128,0.08911573092142741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,16384,1.1817557017008462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,16384,1.9954240163167316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,65536,4.83020273844401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,51200,3.7742602030436196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,12288,0.8800469080607096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,12288,1.4481941223144532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,12288,1.0417813618977865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,10240,0.7652959823608398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,10240,1.2894869486490885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,51200,4.763599141438802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,65536,8.011172485351562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,51200,6.134125773111979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,8192,0.642741330464681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,10240,0.9087157567342123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,8192,0.97762451171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,7168,0.5361525217692058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,6144,0.48998187383015945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,7168,0.8529333114624024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,16384,1.3989898681640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,6144,0.751253318786621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,5120,0.3996469179789225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,5120,0.4828373273213704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,4096,0.5107071876525879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,5120,0.6172874450683594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,65536,6.5947916666666675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,4096,0.312994130452474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,4096,0.38586346308390296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,3584,0.29325440724690754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,3584,0.4471722602844238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,3072,0.24089387257893882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,3584,0.3406208038330078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,7168,0.5935285568237305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,2560,0.21248106956481932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,2560,0.22982826232910156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,2560,0.28786239624023435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,2048,0.16533546447753905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,2048,0.2314901351928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,1536,0.17964266141255697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,1536,0.1382271925608317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,1024,0.12568639914194743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,1024,0.10275519688924153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,8192,0.6545205434163411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,1024,0.12081279754638671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,768,0.099290664990743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,768,0.08475519816080729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,3072,0.37532691955566405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,512,0.07329813639322916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,512,0.0661354660987854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,512,0.09082132975260417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,256,0.04748800198237101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,256,0.05826239983240763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,128,0.042947200934092204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,128,0.052945065498352054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,128,0.06934933662414551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,64,0.03901120026906331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,6144,0.509659735361735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,64,0.0529141346613566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,12288,32,0.044189866383870444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,12288,32,0.05307093461354574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,3072,0.2674954732259115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,2048,0.1962272008260091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,1536,0.1584810733795166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,768,0.10891733169555665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,12288,256,0.07472000122070313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,16384,0.9872490564982096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,16384,1.601873016357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,65536,4.169538116455078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,51200,3.2383092244466147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,16384,1.1313706715901692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,12288,0.7923861185709635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,12288,1.2290037790934245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,51200,3.799246978759766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,10240,0.6593781153361002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,65536,5.362759399414062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,51200,5.053517659505209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,10240,1.0549930572509765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,10240,0.737556266784668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,8192,0.8058261235555013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,8192,0.5203114509582519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,7168,0.4923253377278646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,65536,6.483058166503906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,7168,0.697217051188151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,7168,0.5199594815572103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,6144,0.41129385630289717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,6144,0.4403200149536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,6144,0.613975461324056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,5120,0.5143957455952962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,5120,0.3456960042317708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,4096,0.2911893208821615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,4096,0.38689066569010416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,3584,0.24706880251566568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,3584,0.3688181241353353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,3072,0.3150069236755371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,3072,0.2131509304046631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,3072,0.2609482606252035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,2560,0.1866378625233968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,2560,0.2794303894042969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,2560,0.21113386154174804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,2048,0.20205866495768227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,2048,0.17519040107727052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,2048,0.18050452868143718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,1536,0.1569930712381999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,1536,0.1396074612935384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,1024,0.11569920380910237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,1536,0.14414079984029132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,1024,0.09398079713185628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,768,0.09170880317687988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,1024,0.11743040084838867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,768,0.07854186693827311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,12288,0.8256714502970377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,512,0.06611946821212769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,512,0.0626250664393107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,256,0.04382933378219604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,512,0.09466773668924967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,256,0.05198506514231364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,256,0.06662933429082235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,128,0.0387935996055603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,128,0.046904532114664714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,64,0.033053867022196454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,128,0.060670932133992515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,10240,32,0.033642665545145674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,64,0.04733866850535075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,10240,32,0.04760640064875285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,8192,0.5466239929199219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,5120,0.35970134735107423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,4096,0.29272212982177737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,3584,0.2580479939778646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,10240,768,0.09194133281707764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,65536,3.4229281107584635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,51200,2.6808021545410154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,16384,1.2893034617106118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,65536,4.276313781738281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,16384,0.8709215799967447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,51200,2.988262430826823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,51200,3.960529073079427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,16384,0.9594367980957031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,65536,5.076123555501303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,12288,1.0134314854939779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,12288,0.7390431722005208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,12288,0.7087253570556641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,10240,0.5332799911499023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,10240,0.5771125157674153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,10240,0.8166240056355795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,8192,0.687940279642741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,8192,0.4847861289978027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,8192,0.4412896156311035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,7168,0.5999253590901692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,7168,0.4133098602294922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,6144,0.5131989479064941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,6144,0.3412864049275716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,5120,0.2905909220377604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,6144,0.3897280057271322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,4096,0.32574294408162435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,4096,0.264627202351888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,4096,0.2461301326751709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,3584,0.27205012639363607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,3584,0.21698880195617676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,3072,0.17819093068440756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,3072,0.2710986773173014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,3072,0.19837013880411786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,2560,0.20551039377848307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,2560,0.15013972918192547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,2560,0.1814677397410075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,2048,0.16208426157633465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,2048,0.13324053287506105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,2048,0.14200852711995443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,1536,0.1308778683344523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,1536,0.10645546913146972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,1024,0.08669866720835367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,1024,0.0843552033106486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,768,0.0696618636449178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,7168,0.4074080149332683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,1024,0.09437440236409506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,768,0.06885973612467447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,512,0.05184853474299113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,768,0.07803733348846435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,512,0.05367146730422974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,256,0.03438933293024699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,512,0.068067200978597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,256,0.04466346502304077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,128,0.03060479958852132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,64,0.027153066794077557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,128,0.049371735254923506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,5120,0.4107381184895833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,8192,32,0.028435200452804565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,64,0.0413642684618632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,5120,0.3251466751098633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,32,0.041569066047668454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,256,0.055061332384745275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,8192,128,0.04160106579462687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,3584,0.21857919692993164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,8192,1536,0.11267413298288982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,65536,3.110332743326823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,16384,1.0530890782674154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,16384,0.7674986521402996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,51200,2.425006866455078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,51200,3.8114603678385413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,16384,0.8411754608154297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,51200,2.746490732828776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,12288,0.906924819946289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,10240,0.7246335983276367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,65536,4.793775939941407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,10240,0.5388554890950521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,8192,0.6078261057535808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,8192,0.4236703872680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,8192,0.430295467376709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,7168,0.37518720626831054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,7168,0.5358847935994466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,6144,0.313150946299235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,12288,0.596013895670573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,6144,0.42653973897298175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,6144,0.3518815994262695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,5120,0.2532490730285645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,5120,0.33902934392293294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,4096,0.2934314727783203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,10240,0.5237386703491211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,4096,0.20644159317016603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,4096,0.24080212910970053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,3584,0.18160959879557292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,3584,0.2582080046335856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,3584,0.19615999857584637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,3072,0.21149226824442544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,3072,0.19302186965942383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,2560,0.17426026662190755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,65536,3.3909077962239587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,2560,0.1518869400024414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,2048,0.14205013910929362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,2048,0.13295466899871827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,2048,0.12479893366495769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,1536,0.1200757344563802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,1536,0.09346026579538981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,1024,0.07786773045857748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,1024,0.07274879614512125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,1024,0.08083626429239908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,768,0.06201066573460897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,12288,0.6114453633626302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,768,0.06146346728006998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,768,0.07372799714406332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,512,0.04695680141448975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,512,0.05379413366317749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,512,0.05841066837310791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,3072,0.15841174125671387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,256,0.03486506541570027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,256,0.04056426684061686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,128,0.026469333966573076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,128,0.03788160085678101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,64,0.023642667134602866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,64,0.038525867462158206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,7168,32,0.02537386616071065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,7168,32,0.03869119882583618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,7168,0.3695253372192383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,5120,0.2631072044372559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,2560,0.14639573097229003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,1536,0.10272853374481201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,256,0.048427732785542806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,7168,128,0.045178667704264326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,65536,2.9103988647460937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,65536,3.0818356831868488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,16384,0.8981365203857422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,16384,0.6961973190307618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,65536,4.034112040201823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,12288,0.5673866907755534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,51200,2.1809547424316404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,12288,0.7955498377482096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,51200,3.2570228576660156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,10240,0.6056149164835612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,10240,0.4713631947835286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,10240,0.45942827860514324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,8192,0.49664106369018557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,8192,0.3691103935241699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,7168,0.4198261260986328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,7168,0.3428544044494629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,7168,0.33416318893432617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,6144,0.35044587453206383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,6144,0.29180374145507815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,6144,0.2833813349405924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,5120,0.3086037317911784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,5120,0.24644160270690918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,5120,0.24291946093241373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,4096,0.23989653587341309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,4096,0.20843626658121744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,3584,0.20981973012288413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,3584,0.1758570671081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,3584,0.1789354642232259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,3072,0.1866773287455241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,3072,0.15206294059753417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,2560,0.15901333491007488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,2560,0.12924906412760417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,2560,0.13855039278666179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,2048,0.12537919680277507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,2048,0.11319786707560223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,2048,0.11057173411051432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,1536,0.0981653372446696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,1536,0.09101973374684652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,16384,0.7018400192260742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,1024,0.06849280198415121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,1024,0.08365546862284343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,768,0.06734399795532227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,768,0.06399573485056559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,12288,0.5229599952697754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,512,0.04201279878616333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,512,0.04678613344828288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,51200,2.3109865824381512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,256,0.03673280080159505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,8192,0.3538506507873535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,256,0.044801068305969236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,128,0.026927999655405682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,256,0.04127466678619385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,128,0.03674026727676392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,64,0.021458133061726888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,64,0.03660373290379842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,6144,32,0.02206826607386271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,6144,32,0.035854931672414145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,4096,0.1885141372680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,3072,0.14645867347717284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,1536,0.08722453117370606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,1024,0.06827413241068522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,768,0.060296531518300375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,512,0.05039893388748169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,65536,2.457733408610026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,6144,128,0.03852479855219523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,16384,0.7658175786336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,65536,2.5193631490071615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,65536,3.317879486083984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,51200,1.9345205942789714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,16384,0.6330378850301106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,16384,0.5851968129475911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,12288,0.6301888147989909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,12288,0.5031711896260579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,12288,0.4600672086079915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,51200,2.707563781738281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,10240,0.3926495869954427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,10240,0.5057888031005859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,8192,0.4027743975321452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,8192,0.3417621294657389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,8192,0.33205652236938477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,7168,0.34305171966552733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,7168,0.3025055885314941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,6144,0.29372587203979494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,6144,0.24489280382792153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,6144,0.2581834634145101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,5120,0.2510634740193685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,5120,0.21615146001180013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,4096,0.1991754690806071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,4096,0.18177493413289386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,3584,0.1748842716217041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,3584,0.16169385910034179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,3072,0.15309759775797527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,3072,0.13600746790568036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,51200,1.9681386311848956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,2560,0.16253652572631835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,2560,0.1132853349049886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,10240,0.37699413299560547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,2048,0.10500906308492024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,2048,0.09710720380147299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,1536,0.08124266465504965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,7168,0.26557013193766277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,1536,0.07828906377156576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,1024,0.05757973194122314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,1024,0.060737065474192296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,5120,0.1959829330444336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,768,0.046197334925333663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,768,0.05108160177866618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,4096,0.16114986737569173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,768,0.05185173352559408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,3584,0.1432970682779948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,512,0.03526293436686198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,512,0.04336959918340047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,256,0.024460800488789878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,512,0.041894400119781496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,256,0.033598931630452473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,128,0.019258666038513183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,256,0.03585813442866008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,64,0.01739733417828878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,128,0.031498666604359946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,5120,32,0.017927465836207072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,64,0.03150720000267029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,5120,32,0.03153493404388428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,3072,0.12700053056081134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,2560,0.10982080300649007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,2048,0.09129172960917155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,1536,0.07507306734720866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,65536,2.0598730723063152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,1024,0.05942186514536539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,65536,2.5890858968098955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,16384,0.6160800298055013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,16384,0.5298186620076497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,5120,128,0.032979200283686325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,12288,0.483462397257487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,12288,0.40573441187540693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,51200,1.6283466339111328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,51200,2.1078868865966798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,10240,0.41187734603881837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,10240,0.34130452473958334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,8192,0.3494410514831543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,8192,0.29437761306762694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,7168,0.2971861203511556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,7168,0.25547520319620765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,6144,0.2255253314971924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,6144,0.21970240275065103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,65536,3.0204139709472657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,5120,0.21520105997721353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,16384,0.6442666371663411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,12288,0.47946027119954426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,5120,0.19065279960632325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,4096,0.15657706260681153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,4096,0.15626452763875326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,10240,0.4078197479248047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,3584,0.1374623934427897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,3584,0.1296672026316325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,8192,0.3211359977722168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,51200,2.2677759806315105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,3072,0.11908799807230633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,3072,0.11420799891153972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,3072,0.13357866605122884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,2560,0.1010101318359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,2560,0.09908373355865478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,2048,0.0826047976811727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,7168,0.28937387466430664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,1536,0.0646559993426005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,1536,0.06970026493072509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,6144,0.2396618684132894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,1024,0.04639039834340413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,1024,0.0548746665318807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,2048,0.08445759614308676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,768,0.0375765323638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,5120,0.20021546681722008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,4096,0.1649386723836263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,768,0.04551146825154622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,512,0.028985599676767986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,512,0.03801920016606648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,3584,0.14442133903503418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,256,0.020667733748753865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,256,0.03079040050506592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,2560,0.10819306373596191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,128,0.015742933750152587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,2048,0.08717439969380697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,128,0.028114134073257448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,64,0.014076800147692362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,1536,0.06512106657028198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,4096,32,0.014393599828084311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,32,0.027592533826828004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,1024,0.051444268226623534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,4096,64,0.028153600295384724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,768,0.04058666626612346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,65536,1.9563583374023437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,65536,2.433825174967448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,512,0.033061333497365314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,256,0.02695786754290263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,4096,128,0.022694400946299233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,16384,0.5922527949015299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,16384,0.5037962595621746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,51200,1.7694272359212238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,12288,0.41118399302164715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,12288,0.40570987065633135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,65536,2.3285781860351564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,10240,0.3839808146158854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,10240,0.3460864067077637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,8192,0.2900373458862305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,51200,1.752748743693034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,8192,0.27347307205200194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,7168,0.2568938732147217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,7168,0.24680533409118652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,6144,0.2148149331410726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,6144,0.2063434600830078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,5120,0.17360960642496745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,5120,0.17382826805114746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,51200,1.5326464335123697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,4096,0.13924266497294108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,4096,0.13993813196818033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,4096,0.14139626820882162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,3584,0.12365333239237468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,3584,0.12207573254903156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,16384,0.5696906407674154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,3072,0.11306347052256267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,3072,0.11209920247395833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,12288,0.42745812733968097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,2560,0.0945034662882487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,10240,0.3578037261962891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,2560,0.0939253330230713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,2560,0.09594559669494629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,2048,0.07435306708017984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,8192,0.28146772384643554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,2048,0.08158613046010335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,2048,0.07710826396942139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,1536,0.06656320095062256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,1024,0.04314239819844563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,7168,0.25258560180664064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,1024,0.05299306710561117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,768,0.0357151985168457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,768,0.0443391998608907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,6144,0.2084714730580648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,768,0.037273601690928145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,512,0.028068266312281293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,512,0.030186667044957476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,256,0.019782400131225585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,256,0.029833600918451948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,5120,0.17629119555155437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,128,0.015244799852371215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,128,0.027062400182088213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,1536,0.061187199751536046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,64,0.01328000028928121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,64,0.027831466992696126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3584,32,0.013698133826255798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,32,0.027793065706888838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3584,512,0.03727039893468221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,3584,0.12478079795837402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,3072,0.10866666634877523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,1024,0.04609920183817546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,1536,0.05856320063273111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,65536,2.1362325032552083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,65536,1.8549184163411456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,256,0.024434133370717367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,51200,1.5994271596272787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,16384,0.5244074821472168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3584,128,0.020487467447916664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,16384,0.47412799199422195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,12288,0.40079466501871747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,12288,0.3874506632486979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,10240,0.30299307505289713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,51200,1.4092703501383463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,10240,0.3397280057271322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,8192,0.25439680417378746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,10240,0.33127787907918294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,8192,0.24397546450297036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,7168,0.22523840268452963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,7168,0.2178335984547933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,7168,0.227730131149292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,6144,0.18363626797993976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,6144,0.1892159938812256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,5120,0.1530794620513916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,5120,0.16133119265238444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,65536,2.176996358235677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,4096,0.13000319798787435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,16384,0.5119039853413899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,4096,0.13722559611002605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,3584,0.11122346719106038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,12288,0.38041814168294275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,3584,0.11758506298065186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,3584,0.11757013003031414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,3072,0.09382399717966715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,3072,0.1017301321029663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,51200,1.6754688262939452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,2560,0.08082666397094726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,8192,0.26393280029296873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,2560,0.0956821362177531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,2560,0.08591039975484213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,2048,0.06744320392608642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,1536,0.05211520195007324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,2048,0.08207573095957438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,1536,0.061345068613688145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,1024,0.03777173360188802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,1024,0.047186132272084555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,1024,0.04190933307011922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,6144,0.1937173366546631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,768,0.031049599250157673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,768,0.039288532733917234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,512,0.03334826628367106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,4096,0.13253333568572997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,5120,0.16155840555826823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,256,0.01669973333676656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,256,0.029253333806991577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,128,0.013075199723243714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,128,0.02641493280728658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,512,0.023218133052190146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,64,0.011404800415039062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,64,0.025919999678929644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,3072,32,0.012714667121569314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,3072,32,0.025788799921671553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,3072,0.09894826412200927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,2048,0.07046079635620117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,1536,0.053142400582631436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,768,0.03325226704279582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,512,0.0269813338915507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,65536,1.634225082397461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,256,0.021539199352264404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,16384,0.41513706843058273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,3072,128,0.018148267269134523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,65536,1.6106527964274089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,16384,0.4134527842203776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,12288,0.2807285308837891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,51200,1.2383295694986978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,51200,1.2604202270507812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,10240,0.2357685407002767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,12288,0.3190261205037435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,10240,0.2959957440694173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,8192,0.18979627291361492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,8192,0.2208415985107422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,7168,0.16720639864603679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,7168,0.19043839772542318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,6144,0.1419317404429118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,6144,0.16557226181030274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,6144,0.16480426788330077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,5120,0.11922880013783772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,5120,0.1405898729960124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,16384,0.4294944127400716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,4096,0.09803733030954996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,12288,0.32277441024780273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,65536,1.822306187947591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,4096,0.11682986418406169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,51200,1.435534922281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,10240,0.27522347768147787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,3584,0.10305813153584797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,3072,0.07497706413269042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,3072,0.09056106408437094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,8192,0.22215360005696616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,2560,0.0637824018796285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,7168,0.19510399500528972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,2560,0.07738880316416422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,2560,0.07606613636016846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,2048,0.05216319958368937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,2048,0.06533013184865316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,1536,0.04037546714146932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,1536,0.052510933081309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,1024,0.02895466685295105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,1024,0.04058666626612346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,3584,0.09447466532389323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,768,0.02399786710739136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,5120,0.138590939839681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,768,0.03398186763127645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,512,0.01826453407605489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,512,0.028493867317835493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,512,0.023770666122436522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,256,0.01248426636060079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,256,0.02440426747004191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,4096,0.1145407994588216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,3584,0.10140906969706218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,3072,0.08640746275583902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,128,0.009698133667310078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,128,0.022182399034500123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,64,0.008217599987983704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,64,0.02394346594810486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2560,32,0.011266133189201355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,1536,0.04668266773223877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2560,32,0.02392746607462565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,1024,0.03668800195058187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,768,0.02909226616223653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,65536,1.2491563161214194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,2048,0.06153066555658976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,256,0.01913493275642395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,65536,1.4735018412272134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,16384,0.2965834617614746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,16384,0.3798176129659017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,12288,0.22813760439554848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,51200,1.1636576334635416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,12288,0.2929727872212728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,10240,0.1910154660542806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2560,128,0.015786666671435037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,10240,0.24416426022847496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,10240,0.2353823979695638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,8192,0.15427947044372559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,51200,0.9337088267008463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,8192,0.19770132700602214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,7168,0.13516586621602375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,6144,0.11702506542205811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,7168,0.17337066332499187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,6144,0.15711466471354169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,5120,0.09847253163655599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,5120,0.1341056029001872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,65536,1.4193077087402344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,4096,0.08371946811676026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,16384,0.36119467417399087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,4096,0.11421653429667156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,3584,0.07276586691538492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,51200,1.1481685638427734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,3584,0.0967029333114624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,3072,0.06104960044225057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,3072,0.08330132961273193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,12288,0.2743722597757975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,2560,0.05172479947408041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,2560,0.07110293706258139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,8192,0.18639893531799318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,7168,0.163371737798055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,2048,0.0589845339457194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,6144,0.13669546445210773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,1536,0.03219093283017476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,1536,0.04610666831334432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,5120,0.11633066336313884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,1024,0.02353386680285136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,1024,0.03520426750183105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,1024,0.03130346735318502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,768,0.01936960021654765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,768,0.030300800005594892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,2048,0.041677868366241454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,512,0.014604799946149192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,512,0.02513813376426697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,4096,0.09796053568522135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,256,0.010615467031796774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,256,0.02230400045712789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,3584,0.08545493284861247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,128,0.00827946662902832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,128,0.02031146685282389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,3072,0.07056852976481119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,64,0.007225599884986877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,64,0.02040106654167175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,2048,32,0.007657599945863088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,2048,32,0.020118399461110433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,2048,0.052242132027943936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,1536,0.03977813323338826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,65536,1.0000928243001304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,2560,0.0639306664466858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,768,0.024634667237599692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,51200,0.8220042546590169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,65536,1.3392096201578776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,512,0.019986132780710854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,16384,0.24735360145568847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,16384,0.347545591990153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,256,0.01625386675198873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,12288,0.18570346832275392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,51200,1.06234556833903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,10240,0.15584319432576496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,12288,0.2646730740865072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,2048,128,0.013579733173052468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,10240,0.22783466974894204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,8192,0.12208960056304932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,7168,0.10735466480255126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,8192,0.18190293312072753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,7168,0.1615946610768636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,6144,0.09273173014322916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,6144,0.13974612553914387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,65536,1.2037364959716796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,5120,0.08245973587036133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,16384,0.2933312098185221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,51200,0.9150581359863281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,5120,0.12075520356496175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,4096,0.0667189359664917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,12288,0.2245130697886149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,4096,0.09887359937032064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,3584,0.06126720110575358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,3584,0.08488960266113281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,8192,0.14825493494669598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,3072,0.05225173234939575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,7168,0.13242026964823406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,3072,0.07284266948699951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,2560,0.04052799940109253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,2560,0.06161919832229614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,6144,0.1141653299331665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,2048,0.03251413305600484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,2048,0.05073173443476359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,10240,0.18670080502827963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,1536,0.024758400519688924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,1536,0.03914239803949992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,2560,0.05102613369623819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,1024,0.017800533771514894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,1024,0.030014934142430623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,5120,0.09512426853179931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,768,0.01406826674938202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,4096,0.0772991975148519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,768,0.02626240054766337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,512,0.010744532942771912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,768,0.020819199085235596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,512,0.024626133839289348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,256,0.020615466435750327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,3584,0.06919253667195638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,128,0.007514666517575581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,128,0.01970133384068807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,3072,0.058417065938313806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,64,0.006567466755708058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,64,0.01984106699625651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,32,0.006844800213972728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1536,32,0.02060906688372294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,2048,0.04251840114593506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1536,256,0.009054932991663616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,65536,0.6045440038045247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,51200,0.49619626998901367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,1536,0.033225599924723306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,65536,1.157485834757487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,16384,0.15554025967915852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,1024,0.0257205327351888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,16384,0.30113706588745115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,12288,0.11800426642100018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,51200,0.9564885457356771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,512,0.01704746683438619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,12288,0.23028906186421713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,10240,0.09480213324228923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,256,0.013489066561063131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,10240,0.19535892804463703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,8192,0.07713173230489095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,8192,0.16058026949564616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1536,128,0.011545600493748982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,7168,0.14022293090820312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,7168,0.07025067011515299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,6144,0.06020586490631104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,6144,0.12454079786936442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,12288,0.18652693430582684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,5120,0.05021440188090006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,5120,0.10466132958730061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,65536,0.9975765228271485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,4096,0.04122346639633179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,16384,0.2519999980926514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,4096,0.08787413438161215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,3584,0.03614506721496582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,3584,0.0748255968093872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,10240,0.1597962697347005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,3072,0.030551467339197797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,3072,0.06305919885635376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,6144,0.09889600276947022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,2560,0.027081600824991864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,2560,0.05401599804560343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,51200,0.76572265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,7168,0.11697813669840496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,2048,0.02181333303451538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,2048,0.04260160128275554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,1536,0.017219199736913045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,1536,0.03438613414764404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,8192,0.13291412989298504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,1024,0.013079466422398886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,1024,0.028413865963617963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,3072,0.05176000197728475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,768,0.01090773344039917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,768,0.025206400950749712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,1536,0.028972800572713214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,512,0.00881173312664032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,512,0.021291732788085938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,4096,0.0680351972579956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,256,0.00684799998998642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,256,0.01993280053138733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,5120,0.08226133187611898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,128,0.005819733440876007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,128,0.01769599914550781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,2560,0.043986133734385174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,64,0.005172266562779745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,64,0.017962666352589925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,1024,32,0.005745066702365876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,1024,32,0.017812265952428182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,3584,0.05914453268051147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,65536,0.4912714640299479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,1024,0.020950400829315187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,2048,0.036006398995717365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,65536,1.104306157430013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,51200,0.3809173266092936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,51200,0.8591562906901041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,16384,0.12205440203348797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,16384,0.28492374420166017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,12288,0.09134720166524252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,12288,0.21668267250061035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,128,0.009310932954152425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,256,0.01092906693617503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,10240,0.07677866617838541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,10240,0.18420906066894532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,8192,0.061876265207926426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,8192,0.1501237392425537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,512,0.014166399836540222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,7168,0.05412906805674235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,7168,0.13411946296691896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,1024,768,0.017851734161376955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,6144,0.04632960160573323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,6144,0.1167797327041626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,65536,0.8814687728881836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,5120,0.04003626505533854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,5120,0.10159680048624675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,12288,0.1695082664489746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,4096,0.03245546619097392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,4096,0.07995519638061524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,16384,0.226418129603068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,3584,0.029174399375915528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,51200,0.6865002950032552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,3584,0.07083520094553629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,3072,0.0261898676554362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,3072,0.060344533125559485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,10240,0.14563733736673992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,2560,0.02185813387235006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,2560,0.04933120012283325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,6144,0.08816320101420085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,2048,0.018709333737691243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,2048,0.03958400090535481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,7168,0.10503253142038982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,1536,0.014883200327555338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,1536,0.032110933462778726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,8192,0.11803840001424153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,1024,0.01095466713110606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,1024,0.02685439984003703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,3072,0.04716373284657796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,768,0.009388800462086995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,768,0.023642667134602866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,4096,0.06081600189208984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,512,0.00758186678091685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,512,0.02190613349278768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,5120,0.07491733233133951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,256,0.006141866743564606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,256,0.018978132804234823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,3584,0.05373866558074951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,128,0.005121066669623057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,128,0.018151466051737467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,1536,0.024632533391316734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,2560,0.03977919816970825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,64,0.005335466563701629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,768,32,0.005389866729577383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,64,0.017882666985193887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,768,32,0.017644800742467246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,65536,0.32288532257080077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,65536,0.6079274495442708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,2048,0.03199359973271688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,65536,1.014031982421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,1024,0.018891733884811402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,51200,0.25871359507242836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,16384,0.08279146353403727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,51200,0.4546815872192383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,16384,0.155732266108195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,16384,0.26808748245239256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,12288,0.06230400005976359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,12288,0.1225279966990153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,10240,0.052257064978281656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,12288,0.20199146270751953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,51200,0.7956746419270833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,8192,0.04291839996973674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,10240,0.17530986467997234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,8192,0.08021120230356851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,8192,0.14025492668151857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,7168,0.03818026781082153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,6144,0.03328640063603719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,7168,0.07023146947224936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,7168,0.12535040378570556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,6144,0.06107306480407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,6144,0.11094720363616943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,5120,0.028413865963617963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,4096,0.022098133961359658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,5120,0.053291734059651694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,5120,0.09239786465962728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,4096,0.07360533078511557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,3584,0.019692800442377725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,3584,0.06358506679534912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,3584,0.038970665136973066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,3072,0.017359999815622966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,3072,0.05309013525644938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,2560,0.01487573285897573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,256,0.009898666540781658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,512,0.012968533237775168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,2560,0.04380160172780355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,768,0.01586560010910034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,768,128,0.008462933699289958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,2048,0.012762666742006937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,1536,0.010588799913724262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,2048,0.02404906749725342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,2048,0.036111998558044436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,1536,0.019735467433929444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,1536,0.030396799246470135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,1024,0.00842026670773824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,1024,0.015638400117556253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,768,0.007348266740640004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,1024,0.02474986712137858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,768,0.0220853328704834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,512,0.006113066772619883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,10240,0.09961600303649902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,512,0.019643733898798622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,256,0.004965333143870035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,256,0.009351467092831928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,128,0.004564266900221507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,128,0.01739306648572286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,4096,0.043901864687601724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,64,0.004407466451327006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,64,0.016897066434224447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,512,32,0.0045162667830785114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,3072,0.03470720052719116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,32,0.017025067408879598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,65536,0.20128000577290855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,51200,0.16166720390319825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,512,256,0.01812160015106201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,65536,0.536251703898112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,16384,0.05257279872894287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,65536,0.9593738555908203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,16384,0.25216320355733235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,12288,0.04145706494649251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,51200,0.7564853032430012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,12288,0.19384106000264484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,10240,0.03511786858240763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,2560,0.029364265998204547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,10240,0.1642250696818034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,8192,0.028716800610224406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,8192,0.13409600257873536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,768,0.013375999530156455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,7168,0.025835732618967693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,512,0.011725866794586181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,7168,0.11839146614074707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,6144,0.023144533236821495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,512,128,0.008213333288828532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,6144,0.10320639610290527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,5120,0.01646293302377065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,5120,0.08602773348490397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,51200,0.42592639923095704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,16384,0.1381109396616618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,4096,0.012781866391499839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,4096,0.039246932665507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,4096,0.06858133474985759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,3584,0.011704533298810323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,3584,0.05793386697769165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,12288,0.104912002881368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,3072,0.010406399766604107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,3072,0.030313599109649658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,3072,0.047842133045196536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,2560,0.009357866644859315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,2560,0.03892373243967692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,2048,0.008066133161385854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,10240,0.08831040064493814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,2048,0.03137173255284627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,1536,0.007119999825954437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,1536,0.027022933959960936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,1536,0.017900800704956053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,1024,0.005938133100668589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,8192,0.07103253205617269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,1024,0.022856533527374268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,768,0.005277866621812185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,768,0.021342933177947998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,7168,0.06276373465855917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,512,0.004619733492533366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,512,0.01920426686604818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,512,0.010110933581988018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,6144,0.05537279844284058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,256,0.0044170667727788295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,256,0.008152533570925396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,128,0.0037674665451049804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,256,0.018012799819310508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,5120,0.046477866172790525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,128,0.0070816000302632645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,128,0.016484266519546507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,64,0.0034645333886146545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,256,32,0.0035018667578697203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,64,0.01589120030403137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,256,32,0.01579839984575907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,2560,0.02527359922726949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,65536,0.1927797317504883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,51200,0.15286933581034343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,65536,0.5033098538716634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,16384,0.05267946720123291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,65536,0.9495914459228516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,16384,0.24687573115030922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,3584,0.03558506568272908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,12288,0.04188266595204671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,12288,0.1901418685913086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,2048,0.021245867013931274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,10240,0.03057493368784587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,10240,0.1606346607208252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,1024,0.013687466581662497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,8192,0.026177066564559936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,256,768,0.011374933520952861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,8192,0.13166293303171794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,7168,0.02336853345235189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,7168,0.11648000081380208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,7168,0.06111893256505331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,6144,0.020295466979344687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,6144,0.1011349360148112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,51200,0.74650452931722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,51200,0.3984149297078451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,5120,0.013967999815940857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,5120,0.04494293530782063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,5120,0.08464000225067139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,4096,0.06628160079320272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,4096,0.03648746808369954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,16384,0.13502079645792645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,3584,0.010339200496673584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,3072,0.01018773317337036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,3072,0.04556693236033122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,3072,0.02814720074335734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,2560,0.008308266599973042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,2560,0.0379861315091451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,2560,0.02424853245417277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,2048,0.007866666714350382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,2048,0.032041599353154496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,12288,0.10341866811116536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,1536,0.006647466619809468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,1536,0.026978133122126262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,4096,0.011364266276359558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,1024,0.0052255998055140175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,10240,0.08624107042948405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,1024,0.022683733701705934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,1024,0.012426666418711345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,768,0.004846933484077454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,768,0.021356799205144248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,768,0.010419199864069622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,3584,0.05568533341089884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,512,0.004131199916203817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,8192,0.06947413285573324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,512,0.018680532773335777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,256,0.0036170666416486105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,512,0.009282132983207703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,128,0.0032501332461833954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,256,0.017076265811920167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,128,0.015777066349983215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,128,0.006306133170922597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,32,0.003202133377393087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,64,0.015343999862670899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,128,32,0.015438933173815408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,65536,0.18041067123413085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,51200,0.14533440272013348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,6144,0.05272106726964315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,16384,0.04967679977416992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,1536,0.016484266519546507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,16384,0.2478229363759359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,12288,0.03905066649119059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,65536,0.9486623764038086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,10240,0.0321941335995992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,8192,0.024308266242345174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,51200,0.7389237085978191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,12288,0.18836906750996907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,10240,0.15851839383443195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,8192,0.12947733402252198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,6144,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,128,64,0.003142400085926056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,5120,0.011865599950154623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,7168,0.11496213277180989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,4096,0.009986133376757304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,6144,0.09942293167114258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,3584,0.0332042674223582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,3584,0.008455466230710347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,5120,0.08307519753774008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,3072,0.008140799899895985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,3584,0.055421864986419676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,3072,0.04494080146153768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,2560,0.006995200117429097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,2048,0.006318933268388112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,1536,0.005426133175690969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,2048,0.030598400036493938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,1536,0.02722773353258769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,1024,0.022124799092610677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,768,0.004228266576925913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,512,0.0038304001092910765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,768,0.019731199741363524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,256,0.0033226666351159418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,512,0.01844266653060913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,128,0.003110400090614955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,256,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,64,0.0030080000559488933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,128,0.015896532932917276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,2048,0.01994346578915914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,64,0.01498240033785502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,32,0.015202132860819497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,65536,0.1783082644144694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,7168,0.02190720041592916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,51200,0.14335892995198568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,4096,0.06582080125808716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,16384,0.05050986607869466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3585,128,256,0.007201066613197327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,64,2560,0.03633386691411336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,1024,0.004524800181388855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,12288,0.03826346794764201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,10240,0.032755200068155924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,8192,0.02730453411738078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,65536,0.9456341425577799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,12288,0.1889525254567464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,16384,0.24834346771240234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,7168,0.02486506700515747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,10240,0.15994240442911783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,8192,0.12920107046763102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,6144,0.020170666774113975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,7168,0.11464426517486573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,5120,0.01244160036245982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,64,32,0.003011200080315272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,4096,0.010417067011197408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,3584,0.009433600306510925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,3072,0.008317866424719492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,6144,0.09909546375274658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,4096,0.0658357342084249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,5120,0.08328959941864014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,2560,0.007911466558774312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,3584,0.05469226837158203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,3072,0.04532586733500163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,2048,0.006838400165239971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,2560,0.036252800623575845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,1024,0.0044149334232012425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,2048,0.030431999762852983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,1536,0.02608213424682617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,768,0.004113066693147024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,1024,0.023067732652028404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,512,0.0036138666172822317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,256,0.0031690667072931922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,128,0.002935466667016347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,512,0.018148267269134523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,256,0.016481066743532814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,64,0.002845866729815801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,128,0.01569919983545939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,32,0.002902399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,32,0.015198933084805808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,51200,0.7405866622924805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3585,32,1536,0.005367466807365417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,768,0.020231467485427857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3585,32,64,0.015043200055758158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,12288,4.355536905924479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,16384,5.782499186197916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,16384,6.55252176920573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,12288,7.80184326171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,12288,5.160656229654948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,16384,10.369337972005209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,10240,3.4982335408528646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,10240,4.530876668294271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,10240,6.7003733317057295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,8192,3.585505167643229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,8192,4.243627675374349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,8192,6.367532857259115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,7168,3.0718612670898438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,7168,3.2159818013509116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,51200,19.011783854166666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,7168,4.920926920572916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,6144,2.271351369222005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,6144,4.038181304931641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,6144,2.674874623616536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,5120,1.8040245056152344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,5120,2.1298656463623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,4096,1.442312494913737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,5120,3.306089528401693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,4096,1.79857915242513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,4096,2.667200978597005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,3584,1.2868223826090495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,3584,2.357874043782552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,3584,1.6289098103841144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,3072,1.1341227213541667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,51200,23.35185343424479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,3072,2.0209013621012373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,3072,1.3530731201171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,2560,0.9647402445475259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,2048,0.7606762568155925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,2560,1.7896821339925129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,2048,1.3695060729980468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,1536,1.0194933573404947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,2048,0.9664490381876627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,1024,0.4579370816548665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,1536,0.7514783859252929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,1024,0.7021162668863933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,768,0.5304416020711262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,768,0.37178026835123695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,512,0.418837324778239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,768,0.5430528004964192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,512,0.3048992156982422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,256,0.2580832004547119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,256,0.21397013664245607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,512,0.4395882606506348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,128,0.19724799791971842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,256,0.36919145584106444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,1536,0.638211186726888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,128,0.1984608014424642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,64,0.18414932886759441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,32,0.1800106684366862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,64,0.19134292602539063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,65536,32,0.19109333356221517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,2560,1.0693738301595053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,1024,0.5449077606201171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,65536,128,0.32427838643391926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,65536,51200,34.20384521484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,16384,8.084914143880209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,16384,4.401153055826823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,51200,14.800480143229166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,16384,5.923275756835937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,12288,6.769554138183594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,65536,18.77151082356771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,12288,3.527308909098307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,10240,2.7598335266113283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,12288,4.066114044189453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,10240,5.235989379882812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,10240,3.493638356526693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,8192,2.2375775655110677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,51200,19.546476236979167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,8192,4.138082122802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,7168,1.999017588297526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,51200,25.849212646484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,7168,3.531928507486979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,7168,2.489160410563151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,6144,1.7496981302897137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,6144,2.2391957600911456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,6144,3.48817392985026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,5120,1.499667231241862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,8192,2.718236796061198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,5120,1.73875732421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,5120,2.9227765401204424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,4096,1.1548596700032552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,65536,27.387082926432292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,3584,1.124013900756836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,4096,2.16438725789388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,3584,1.8275775909423828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,3072,1.6090805053710937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,3072,0.9261568069458008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,3072,1.1055007934570313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,2560,0.7647253036499023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,2560,1.3424757639567058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,65536,33.51361897786458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,2560,0.8865354537963868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,2048,0.7661333084106445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,2048,1.0372810363769531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,1536,0.8085834503173828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,1536,0.46751359303792317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,1024,0.5250357309977214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,1536,0.6375071843465169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,4096,1.380575942993164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,1024,0.3814634641011556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,768,0.42556479771931965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,512,0.24059093793233238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,768,0.2911434809366862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,512,0.34967679977416993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,768,0.41306454340616866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,3584,1.109107208251953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,256,0.19878826141357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,2048,0.5913333257039388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,256,0.17185707092285157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,128,0.1572917302449544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,512,0.34093332290649414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,128,0.1614016056060791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,256,0.29512958526611327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,64,0.14602986971537274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,51200,32,0.14215146700541179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,64,0.156496000289917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,128,0.25554986000061036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,51200,32,0.15490773518880208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,51200,1024,0.43010346094767254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,16384,2.6866559346516925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,16384,1.5522496541341146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,16384,1.7017706553141276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,51200,4.805323791503906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,51200,5.54027099609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,12288,1.162072499593099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,12288,2.0031380971272785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,65536,6.354082234700521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,10240,0.9630645116170248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,10240,1.654427719116211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,65536,7.682911173502605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,8192,0.7625183741251628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,8192,1.378546142578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,51200,8.516436258951824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,7168,0.7085568110148113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,8192,0.8483530680338541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,7168,1.1472202301025392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,7168,0.8192544301350912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,6144,0.599620246887207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,6144,1.0420031865437827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,5120,0.5462410608927409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,12288,1.317624537150065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,5120,0.8298794428507487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,4096,0.42627731959025067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,4096,0.6845098495483398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,65536,10.36144307454427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,4096,0.5307957331339519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,3584,0.5718538920084636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,3584,0.3694997469584147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,10240,1.0648330688476562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,3072,0.3237119992574056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,3072,0.5007935841878255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,3072,0.39008747736612953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,2560,0.4596138636271159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,2560,0.26922985712687175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,2560,0.3216256141662598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,2048,0.37623573939005533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,2048,0.24621440569559733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,1536,0.25240426063537597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,1536,0.21216425895690919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,1024,0.1779370625813802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,1024,0.14449812571207682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,1024,0.17442132631937662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,768,0.1495584011077881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,768,0.11399679978688557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,512,0.0995807965596517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,768,0.16791253089904784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,512,0.0858837366104126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,512,0.13632960319519044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,256,0.0616703987121582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,6144,0.6305002848307292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,256,0.097652268409729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,128,0.05456533432006836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,128,0.06513919830322265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,64,0.04923733472824097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,64,0.06533653338750203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,5120,0.5198869387308757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,16384,32,0.049755732218424484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,32,0.06862186590830485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,16384,256,0.07154773076375326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,3584,0.3824991861979167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,2048,0.2415679931640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,1536,0.2009119987487793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,16384,128,0.08797547022501627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,16384,2.110341389973958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,16384,1.2541226704915365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,51200,3.865802764892578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,65536,4.982990010579427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,16384,1.3559125264485679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,12288,1.598681640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,51200,4.130269877115885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,12288,0.9179381052652994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,12288,0.9447530746459961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,65536,5.635948689778646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,10240,1.3173600514729817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,10240,0.8271626790364583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,8192,0.692851193745931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,8192,1.06898562113444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,51200,6.358333841959635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,7168,0.6006069183349609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,8192,0.7227893193562825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,7168,0.5821674982706706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,7168,0.9184309641520182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,6144,0.4715135892232259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,6144,0.4977727890014648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,6144,0.7855530420939127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,5120,0.4270517349243164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,5120,0.44404052098592117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,65536,8.103916931152344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,5120,0.6573802947998046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,4096,0.3372917175292969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,4096,0.4732949256896973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,3584,0.30834665298461916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,3584,0.41655785242716475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,3584,0.3233909289042155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,3072,0.4210613250732422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,3072,0.26361813545227053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,2560,0.23456106185913086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,3072,0.29931627909342445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,2560,0.3168181419372559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,2560,0.245194673538208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,2048,0.18636159896850585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,2048,0.2776255925496419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,1536,0.20652480125427247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,2048,0.21607467333475747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,1024,0.13542613983154297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,1536,0.16866666475931805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,1024,0.11795732975006104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,768,0.10675093332926433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,1024,0.13964799245198567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,768,0.0921450694402059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,512,0.08543999989827475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,768,0.11721386909484863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,512,0.07311466534932455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,256,0.0513589342435201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,512,0.10202986399332684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,256,0.0605568011601766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,128,0.042021334171295166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,256,0.07529173692067465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,64,0.03813120126724243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,128,0.07235733668009439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,12288,32,0.03925120035807292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,64,0.052826666831970216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,32,0.05325226783752442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,10240,0.7786901473999024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,1536,0.15139412879943848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,12288,128,0.05302293300628662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,12288,4096,0.3279637336730957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,16384,1.691933822631836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,65536,4.4812255859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,51200,3.2326581319173178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,16384,1.0204479853312174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,51200,5.488837178548177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,12288,0.7723744074503581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,12288,1.229375966389974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,12288,0.7967295964558919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,10240,0.6467050552368164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,10240,0.6695306777954102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,10240,1.0748906453450522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,8192,0.853979746500651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,65536,7.068322245279949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,8192,0.5339850743611654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,8192,0.5789056142171224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,7168,0.46779626210530595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,6144,0.626359494527181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,7168,0.7420746485392253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,6144,0.42577279408772783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,6144,0.40506881078084306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,65536,4.538908894856771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,5120,0.3849280039469401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,5120,0.5261386553446452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,5120,0.34415254592895506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,51200,3.475446319580078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,4096,0.3073888142903646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,4096,0.40549332300821944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,4096,0.32467838923136394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,3584,0.2492074648539225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,3072,0.23340266545613608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,3072,0.31352853775024414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,3072,0.24388480186462402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,2560,0.24935572942097983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,16384,1.037111473083496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,2560,0.1914079984029134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,2048,0.2103381315867106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,2048,0.1566271940867106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,1536,0.1602666695912679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,2048,0.19040106137593588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,1536,0.12912639776865642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,1536,0.13645119667053224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,1024,0.11481386820475262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,1024,0.10023252964019776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,1024,0.1085045337677002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,768,0.08787733713785807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,768,0.08214186827341716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,512,0.06754133701324463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,768,0.09822293122609457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,512,0.0648138682047526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,512,0.08004373709360758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,256,0.045950933297475176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,128,0.036686933040618895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,256,0.053257600466410315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,128,0.04630719820658366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,64,0.032842665910720825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,128,0.061561600367228186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,32,0.033878401915232344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,64,0.04752959807713826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,10240,32,0.047880534331003824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,10240,3584,0.3769354820251465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,7168,0.4526677449544271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,3584,0.2423093318939209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,2560,0.18237226804097492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,10240,256,0.0604149341583252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,16384,1.2274667104085286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,65536,3.5581451416015626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,51200,2.7125279744466146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,65536,3.735247039794922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,16384,0.8472746531168619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,51200,4.098296610514323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,12288,0.6630005518595378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,16384,0.9234709421793619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,12288,0.9751125335693359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,10240,0.7850378672281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,10240,0.5536170959472656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,65536,5.188296508789063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,8192,0.46453119913736984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,8192,0.6726133346557617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,8192,0.48395945231119797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,7168,0.5293845176696778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,7168,0.40794986089070634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,6144,0.49999465942382815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,6144,0.36029440561930337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,6144,0.35681279500325525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,5120,0.308622932434082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,5120,0.3079242706298828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,4096,0.3564586639404297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,4096,0.2456000010172526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,4096,0.25163092613220217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,3584,0.30485760370890297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,3584,0.21272853215535484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,3584,0.21760212580362953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,3072,0.2621685345967611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,3072,0.17991679509480793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,3072,0.1886847972869873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,2560,0.21642773946126304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,2560,0.1549557367960612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,7168,0.4102026621500651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,2048,0.17164373397827148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,2048,0.12612266540527345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,1536,0.12323306401570637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,5120,0.3992608070373535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,1536,0.10197546482086181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,1536,0.10901760260264079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,1024,0.08727146784464518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,12288,0.6460213343302409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,1024,0.07970026334126791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,1024,0.08382826646169027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,768,0.0783402681350708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,51200,2.897469838460286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,512,0.05832639932632446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,768,0.07388160228729249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,256,0.03415786822636922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,10240,0.532257080078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,512,0.07142933209737143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,256,0.045442132155100505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,128,0.030003199974695843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,128,0.04212373495101929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,128,0.05263893206914266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,64,0.02700693408648173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,8192,32,0.02792853315671285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,64,0.042342400550842284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,32,0.043051731586456296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,768,0.06742826302846273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,8192,512,0.06828800042470297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,2560,0.15480000178019207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,2048,0.130841596921285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,8192,256,0.051363201936086025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,65536,3.0881792704264326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,16384,1.099494425455729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,16384,0.7465877532958984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,12288,0.8665376027425131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,51200,2.3618314107259115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,65536,3.3617782592773438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,12288,0.5657429377237956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,65536,4.3987787882486975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,51200,3.4955083211263025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,51200,2.467512512207031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,10240,0.4813066800435384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,12288,0.6106634775797526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,8192,0.5375626881917317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,10240,0.6930485407511393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,8192,0.40355625152587893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,8192,0.45848426818847654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,7168,0.3841877301534017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,7168,0.4830890655517578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,7168,0.36032533645629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,6144,0.43653119405110674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,6144,0.33899946212768556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,5120,0.274676259358724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,5120,0.346560001373291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,5120,0.255948797861735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,4096,0.293719482421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,4096,0.21957546869913735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,4096,0.21528959274291992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,3584,0.19281066258748372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,3584,0.2479680061340332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,3584,0.19258559544881185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,3072,0.2067626635233561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,3072,0.16359999974568684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,3072,0.1642624060312907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,2560,0.14261226654052733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,2560,0.14512532552083332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,2048,0.13696959813435872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,10240,0.5235583941141765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,1536,0.10885333220163981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,2048,0.1206496000289917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,1536,0.09509653250376383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,1024,0.07858880360921225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,1536,0.09796160062154134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,1024,0.07330346902211507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,1024,0.0778112014134725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,768,0.06352959871292115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,512,0.0460426648457845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,768,0.06924160321553549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,512,0.0516543984413147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,256,0.03009066581726074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,512,0.05572906732559204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,256,0.0419648011525472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,128,0.025997867186864216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,6144,0.3143146514892578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,128,0.03963093360265096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,128,0.042998401323954265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,64,0.023918932676315306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,32,0.024037333329518636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,64,0.039603201548258464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,32,0.04043200016021729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,16384,0.7631786982218425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,7168,2560,0.18284160296122234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,2048,0.11582612991333008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,7168,768,0.06163306633631388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,16384,1.0063637415568034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,51200,2.191667175292969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,16384,0.7668415705362956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,65536,2.8266624450683593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,7168,256,0.04558080037434896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,65536,3.011766306559245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,51200,3.1029642740885417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,16384,0.705628776550293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,12288,0.508460807800293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,12288,0.5436960220336914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,12288,0.7717493057250977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,65536,4.135338592529297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,10240,0.6181450525919596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,10240,0.4844095865885417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,8192,0.4951775868733724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,8192,0.3955434799194336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,8192,0.36363093058268225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,7168,0.44821761449178055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,7168,0.3256490707397461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,6144,0.37528746922810874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,6144,0.2870016098022461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,5120,0.32877651850382483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,5120,0.262280527750651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,5120,0.2259199937184652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,4096,0.2637397289276123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,4096,0.20910399754842124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,4096,0.18728426297505696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,3584,0.18072214126586914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,3584,0.16678826014200848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,10240,0.43222185770670574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,3072,0.1889599959055583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,3072,0.15275732676188153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,2560,0.16022186279296874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,2560,0.12842666308085124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,7168,0.335699208577474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,2048,0.12864853541056315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,2048,0.10831147034962971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,51200,2.284253946940104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,1536,0.09697919686635335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,2048,0.10837333202362061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,1024,0.07026027043660482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,1536,0.08705706596374511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,1536,0.10189546744028728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,1024,0.06911786397298178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,768,0.06564693450927735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,768,0.05839146773020426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,768,0.07142720222473145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,512,0.04287039836247762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,512,0.051785600185394284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,512,0.049990399678548174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,256,0.031490133206049604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,3584,0.23292053540547691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,256,0.03831679821014404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,128,0.023509333531061806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,256,0.043601067860921223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,64,0.021709867318471274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,128,0.035913598537445066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,64,0.03570346832275391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,6144,32,0.022980266809463502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,6144,32,0.03571199973424276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,6144,0.262882137298584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,128,0.038819201787312824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,3072,0.1437013308207194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,2560,0.12683200041453044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,65536,2.5043190002441404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,65536,2.445700327555339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,6144,1024,0.06824320157368978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,16384,0.8076288223266601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,12288,0.5685333251953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,16384,0.5879221598307292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,65536,3.352733866373698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,16384,0.6360085169474284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,51200,1.9109418233235675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,51200,2.7520917256673174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,12288,0.5086421330769857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,10240,0.3671317418416341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,10240,0.5239733378092448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,8192,0.3192885398864746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,10240,0.43236265182495115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,8192,0.3330122629801432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,7168,0.31245333353678384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,7168,0.2767541249593099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,6144,0.2480522632598877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,6144,0.24591679573059083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,5120,0.20797972679138182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,5120,0.20148372650146484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,4096,0.1995797316233317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,4096,0.1800554593404134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,4096,0.16178986231486003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,3584,0.17374614079793294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,3584,0.1554911931355794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,3072,0.15528853734334308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,3584,0.14601279894510905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,3072,0.12846506436665853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,3072,0.13773226737976074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,7168,0.35478719075520837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,2560,0.12755839824676513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,2560,0.11583466529846191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,6144,0.29588054021199545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,2560,0.10587200323740642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,2048,0.10408000151316325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,2048,0.0982922633488973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,1536,0.08025493621826171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,1536,0.07974507013956705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,5120,0.2529674688975016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,1024,0.05723520119984945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,1024,0.06181546847025553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,768,0.04723093509674072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,768,0.053020799160003663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,768,0.05279466708501181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,512,0.03598933219909668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,512,0.043969066937764485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,8192,0.3943061192830404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,12288,0.43068908055623367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,256,0.02462079922358195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,51200,1.897944513956706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,256,0.036187732219696046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,128,0.02286400000254313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,128,0.03210773269335429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,128,0.04205333391825358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,64,0.017612799008687337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,5120,32,0.017824000120162962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,64,0.0370741327603658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,5120,32,0.03218773404757182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,2048,0.09111680189768473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,1536,0.07544853687286376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,1024,0.05963519811630249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,65536,2.146514129638672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,512,0.043228801091512045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,65536,2.8114463806152346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,5120,256,0.03532266616821289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,16384,0.642135492960612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,16384,0.5746410369873047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,12288,0.4679541269938151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,51200,1.5625717163085937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,12288,0.36336641311645507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,51200,1.7214251200358075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,65536,2.0866954803466795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,10240,0.3909791946411133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,51200,2.196122741699219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,10240,0.39510720570882163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,8192,0.3324533462524414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,8192,0.25816639264424646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,8192,0.31470187505086267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,7168,0.3189759890238444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,7168,0.2647775967915853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,7168,0.2259157339731852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,6144,0.25961173375447594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,6144,0.22946453094482422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,6144,0.20918720563252768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,5120,0.20163946151733397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,5120,0.170853328704834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,4096,0.1682794729868571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,4096,0.1650826613108317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,4096,0.13111999829610188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,3584,0.154747740427653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,3584,0.1376373291015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,3584,0.1255232016245524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,3072,0.12579840024312336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,3072,0.12790186405181886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,3072,0.101800537109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,2560,0.10936426321665446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,12288,0.43672959009806317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,2560,0.10423040390014648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,2048,0.0874720017115275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,2560,0.09052053292592367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,2048,0.09068373044331869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,1536,0.06930987040201822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,1024,0.050327467918396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,1536,0.07203733126322429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,1024,0.057199998696645105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,768,0.04076053301493327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,1024,0.04899306694666545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,768,0.04872639973958333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,512,0.03208426634470622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,768,0.0420138676961263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,256,0.02188373406728109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,512,0.04077440102895101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,256,0.03296320041020711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,128,0.015095466375350952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,128,0.02744213342666626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,5120,0.19838399887084962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,64,0.01487573285897573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,64,0.028101332982381183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,4096,32,0.015337600310643514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,4096,32,0.028689066569010418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,16384,0.4648736000061035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,1536,0.0648138682047526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,10240,0.29734506607055666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,2048,0.073853866259257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,65536,1.9320031483968099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,65536,2.3647722880045574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,512,0.03475946585337321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,256,0.029156267642974854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,51200,1.7936565399169921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,4096,128,0.026209066311518352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,16384,0.48751039505004884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,16384,0.5476736068725586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,16384,0.41393067042032883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,12288,0.3710495948791504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,12288,0.4359605471293132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,12288,0.3212074597676595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,10240,0.3495125452677409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,10240,0.3114453315734863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,8192,0.29501120249430335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,8192,0.25709439913431803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,51200,1.5047295888264975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,7168,0.2263711929321289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,51200,1.3508096059163412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,8192,0.24035199483235678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,7168,0.2428650697072347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,6144,0.19705173174540203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,7168,0.19679360389709472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,6144,0.2247925281524658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,5120,0.16684586207071941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,5120,0.1731338659922282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,4096,0.14468480745951334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,4096,0.14185066223144532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,3584,0.12326080004374187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,3584,0.12226239840189616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,3584,0.11090453465779622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,3072,0.10291519959767659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,3072,0.11100479761759441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,3072,0.0930506706237793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,2560,0.08755733172098795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,2560,0.09286826451619466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,2560,0.08033280372619629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,2048,0.07129600048065185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,2048,0.07944426536560059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,2048,0.0686581293741862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,1536,0.0557151993115743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,1536,0.05666026671727499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,1536,0.06487253506978354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,1024,0.04129493236541748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,65536,1.7397045135498046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,1024,0.05037973324457804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,1024,0.04456426699956258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,768,0.04026666482289632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,768,0.04327679872512817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,512,0.0308351993560791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,512,0.039768532911936445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,512,0.03628480037053426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,256,0.021683200200398763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,256,0.029397332668304445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,128,0.01420906682809194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,128,0.02659306724866231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,128,0.02323946754137675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,64,0.012142933408419291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,64,0.02654186685880025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3584,32,0.012645333011945089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3584,32,0.027063467105229694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,10240,0.268559996287028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,6144,0.1689184029897054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,5120,0.14264960289001466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,4096,0.11535360018412273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,65536,2.044777552286784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,65536,1.8556512196858723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,51200,1.6198720296223956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,768,0.038007465998331706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,16384,0.4801685333251953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,51200,1.4423168182373047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3584,256,0.025969066222508747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,12288,0.353219191233317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,16384,0.4806976000467936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,12288,0.3765343983968099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,10240,0.29819307327270506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,10240,0.3231328010559082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,8192,0.23962987263997398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,8192,0.25144319534301757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,7168,0.21142400105794273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,7168,0.2237781365712484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,6144,0.1781450589497884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,6144,0.1943936030069987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,12288,0.35512641270955403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,5120,0.1576192061106364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,5120,0.1634453296661377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,65536,1.997659683227539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,6144,0.17123732566833497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,4096,0.14932266871134442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,4096,0.13466134071350097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,16384,0.47048212687174484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,3584,0.11279040177663166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,10240,0.2995445251464844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,3584,0.12472426891326904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,3072,0.09898880322774252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,3072,0.10717013676961262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,2560,0.08601813316345215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,2560,0.09222826957702637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,51200,1.4798805236816406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,2048,0.08086293538411458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,7168,0.20739946365356446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,2048,0.08786453406016032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,1536,0.06387519836425781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,1536,0.06885973612467447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,8192,0.23021440505981444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,1024,0.039324800173441574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,1024,0.048520533243815105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,3072,0.091102933883667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,768,0.03211733301480611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,768,0.04126186768213908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,1536,0.048571733633677165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,512,0.02481600046157837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,512,0.034884266058603924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,4096,0.12350506782531738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,256,0.017237333456675212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,256,0.02769920031229655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,2560,0.0795733372370402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,5120,0.14488639831542968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,128,0.01349546710650126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,128,0.026273065805435182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,1024,0.04040426810582479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,3584,0.10398506323496501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,64,0.01160426636536916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,64,0.02648106614748637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,3072,32,0.01211840013662974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,3072,32,0.026219733556111652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,2048,0.06244266827901205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,65536,1.6983754475911457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,65536,1.64771728515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,51200,1.264846928914388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,51200,1.2925610860188803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,16384,0.43731091817220047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,16384,0.42512747446695964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,12288,0.29284159342447913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,128,0.016062933206558227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,12288,0.3336149215698242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,256,0.01912213365236918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,10240,0.24528640111287436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,512,0.024999467531840007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,10240,0.27618452707926433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,8192,0.19636799494425455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,8192,0.2250389258066813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,7168,0.17867093086242675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,7168,0.20917654037475586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,3072,768,0.0303327997525533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,65536,1.5664789835611979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,6144,0.15185492833455402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,6144,0.18720320065816243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,5120,0.1320789337158203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,5120,0.15140053431193035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,51200,1.2183167775472006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,4096,0.11390293439229329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,4096,0.1317354679107666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,16384,0.3722496032714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,3584,0.09305919806162516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,3584,0.11052052974700928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,3584,0.09099733034769694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,3072,0.0807370662689209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,12288,0.28225491841634115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,3072,0.09467840194702148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,2560,0.06819093227386475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,10240,0.24846506118774414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,2560,0.08197440306345621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,2048,0.05560533205668131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,8192,0.18973654111226398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,2048,0.06931520303090413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,1536,0.0439082662264506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,1536,0.055349334081014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,6144,0.15061440467834472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,1024,0.03132266600926717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,1024,0.04199039936065674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,7168,0.17098666826883951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,768,0.025847466786702473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,768,0.03550826708475749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,5120,0.12395199934641521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,512,0.019542400042215982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,512,0.030405332644780476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,4096,0.10068693161010742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,256,0.013409066200256347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,256,0.024578134218851723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,3072,0.07507946491241455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,128,0.010436266660690308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,128,0.02215893268585205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,2560,0.06704959869384766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,64,0.008637866377830506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,64,0.02244053284327189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2560,32,0.008973866701126099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,2048,0.05501759847005209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,1536,0.04306346575419108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,1024,0.03556160132090251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,768,0.026980266968409224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,65536,1.3143627166748046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2560,32,0.022231467564900718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,512,0.02153173287709554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,65536,1.509786605834961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,16384,0.3173962593078613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,256,0.016459733247756958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,16384,0.3914431889851888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,12288,0.24172266324361166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,51200,1.187118911743164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,12288,0.2988095919291178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,10240,0.19964906374613445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2560,128,0.014105600118637086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,10240,0.2521087964375814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,8192,0.16095679601033527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,8192,0.20471253395080566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,7168,0.14314346313476561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,7168,0.18095359802246094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,51200,1.055964787801107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,6144,0.12435306708017986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,6144,0.1573834737141927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,51200,1.1304896036783854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,65536,1.428652826944987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,16384,0.34985920588175456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,5120,0.11770346959431965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,12288,0.2701354662577311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,4096,0.09578879674275717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,5120,0.1452778657277425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,4096,0.12027839819590251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,10240,0.22963199615478516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,3584,0.08527146975199382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,3584,0.09666133721669515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,3584,0.08925546805063883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,3072,0.06367146571477254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,3072,0.08786773681640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,8192,0.18228160540262858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,2560,0.05543253421783447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,2560,0.07272000312805176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,2048,0.04464853207270304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,2048,0.06098560094833374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,7168,0.16030826568603515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,2048,0.051742935180664064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,1536,0.03522666692733765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,1536,0.048053332169850665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,1024,0.025561600923538208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,1024,0.03637760082880656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,6144,0.1377354621887207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,768,0.02109653353691101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,768,0.0311573326587677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,512,0.015770666797955833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,5120,0.1163808027903239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,4096,0.0948309342066447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,512,0.02675093412399292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,256,0.011242666840553283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,256,0.01619733373324076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,256,0.023372799158096313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,128,0.008433066805203756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,128,0.02114773392677307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,128,0.01381119986375173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,64,0.007235200206438701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,2048,32,0.007795199751853943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,64,0.02079253395398458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,3072,0.07048213481903076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,2048,32,0.020601600408554077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,2560,0.062235732873280845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,1536,0.03930240074793498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,51200,0.7330165227254232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,1024,0.030986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,16384,0.22913813591003418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,65536,1.31070187886556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,768,0.024675200382868447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,12288,0.17661546071370443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,16384,0.3481055895487467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,51200,1.029531733194987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,12288,0.25892693201700845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,10240,0.14596053759257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,10240,0.21822826067606607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,8192,0.11889173189798992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,65536,0.96146240234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,8192,0.17797759373982747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,7168,0.10444800059000652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,7168,0.15774186452229816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,2048,512,0.01990293264389038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,6144,0.0885802666346232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,6144,0.13554773330688477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,12288,0.21613334019978842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,5120,0.07430506547292073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,5120,0.11554986635843914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,65536,1.107966995239258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,4096,0.06140480041503906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,16384,0.2606613318125407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,4096,0.09945387045542399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,3584,0.05435520013173421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,3584,0.08741652965545654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,10240,0.17548373540242512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,3072,0.04587093194325765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,6144,0.10417173703511555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,3072,0.07236906687418619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,2560,0.03892373243967692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,2560,0.06150399843851725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,51200,0.8766453425089518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,2048,0.03142720063527425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,2048,0.050426665941874185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,7168,0.12165653705596924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,1536,0.02494186758995056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,1536,0.03918399810791016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,8192,0.13633920351664225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,1024,0.017568000157674155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,1024,0.0324458658695221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,3072,0.053667199611663816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,768,0.014120533068974813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,768,0.027847466866175334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,4096,0.07060373624165853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,1536,0.030943999687830608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,512,0.010632533828417461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,512,0.02435413400332133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,256,0.00872213343779246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,256,0.021347200870513915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,5120,0.08722986380259196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,128,0.007208533088366191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,128,0.01928106745084127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,2560,0.04750293493270874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,64,0.0060576001803080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,64,0.01967680056889852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1536,32,0.006433066725730896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1536,32,0.019248000780741372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,3584,0.06254506508509318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,1024,0.024589866399765015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,65536,0.6296533584594727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,2048,0.03984853426615397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,51200,0.4873269399007161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,65536,0.9949151992797851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,65536,1.1705525716145835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,16384,0.15930986404418945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,51200,0.9368042627970377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,16384,0.30564587910970054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,12288,0.12046826680501302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,12288,0.2303797403971354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,12288,0.17960960070292156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,10240,0.10141119956970215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,10240,0.19448960622151693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,8192,0.08203413486480712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,8192,0.16117119789123535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,256,0.012165333827336628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,128,0.010484266281127929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,7168,0.0728437344233195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,7168,0.1427680015563965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,6144,0.061800531546274816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,6144,0.12503146330515544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,512,0.015729066729545594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,5120,0.051660799980163576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,5120,0.10695466995239258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,51200,0.7422346750895182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,4096,0.04316800038019816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,16384,0.24224106470743814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,4096,0.08620800177256266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,3584,0.038447999954223634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,3584,0.0757258653640747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1536,768,0.01887680093447367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,3584,0.05763413508733114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,3072,0.033131732543309526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,3072,0.06517759958902994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,2560,0.028535467386245728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,2560,0.05460053284962972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,10240,0.15482239723205565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,2048,0.024131200710932412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,2048,0.04452799956003825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,8192,0.1289749304453532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,1536,0.01867626706759135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,1536,0.035426131884257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,7168,0.11231786410013836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,1024,0.014089600245157877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,1024,0.027959465980529785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,6144,0.09655679861704508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,768,0.011785599589347839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,768,0.024129066864649454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,5120,0.08085866769154867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,512,0.008308266599973042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,512,0.021236266692479452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,4096,0.06645013491312662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,256,0.006222933530807495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,256,0.01854506731033325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,3072,0.05035200119018555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,128,0.005780266722043356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,128,0.0184714674949646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,2560,0.042727466424306235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,64,0.0051807999610900875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,64,0.017552000284194947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,1024,32,0.005336533486843109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,1024,32,0.018026665846506754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,2048,0.03564480145772298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,65536,0.4761631965637207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,1536,0.028130133946736652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,51200,0.37742398579915365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,1024,0.020467199881871543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,768,0.01761386593182882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,16384,0.12500373522440594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,512,0.014069333672523499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,51200,0.8764117558797201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,16384,0.2850709279378255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,12288,0.09609706401824951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,256,0.011013333002726238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,10240,0.07976426283518473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,12288,0.21654826800028482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,65536,1.107034683227539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,10240,0.1844853401184082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,1024,128,0.009297066926956176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,8192,0.06518506606419881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,7168,0.05736853281656901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,8192,0.15152853329976398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,7168,0.13541760444641113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,6144,0.04974933465321858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,6144,0.11842026710510253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,65536,0.8550613403320313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,5120,0.04405866861343384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,5120,0.10483946800231933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,51200,0.6478591918945312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,4096,0.03470079898834229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,4096,0.08226133187611898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,16384,0.21490240097045898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,3584,0.02986026604970296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,12288,0.16357226371765138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,3584,0.07106133302052817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,3072,0.026348799467086792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,3072,0.060271998246510826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,10240,0.14037440617879232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,8192,0.11222506364186605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,7168,0.09903999964396158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,2560,0.022685867547988892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,2560,0.04996159871419271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,2048,0.018771199385325114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,2048,0.04007786512374878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,6144,0.08489493529001871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,1536,0.015077333648999533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,1536,0.03294826745986938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,5120,0.07157546679178874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,1024,0.011494400103886922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,1024,0.02686186631520589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,4096,0.05821226835250855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,768,0.009594666957855224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,768,0.02363626758257548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,768,0.015547733505566916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,512,0.00787306676308314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,3584,0.05151146650314331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,512,0.021022933721542358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,256,0.006135466694831848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,3072,0.044698667526245114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,256,0.018911999464035035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,128,0.004869333406289419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,128,0.017334399620691936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,64,0.0046528001626332605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,2560,0.0382805347442627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,64,0.017569067080815633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,768,32,0.004761599997679392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,2048,0.03148693243662516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,768,32,0.016951467593510947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,65536,0.3265023867289225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,1536,0.02422719995180766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,51200,0.26626240412394203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,65536,1.0187967936197917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,16384,0.08462719917297364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,51200,0.8032522837320963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,16384,0.15420160293579102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,16384,0.26452372868855795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,12288,0.06495999892552694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,12288,0.11729599634806316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,12288,0.2024064064025879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,10240,0.05456639925638834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,1024,0.018413867553075156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,8192,0.044215468565622966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,10240,0.09795839786529541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,10240,0.1711466630299886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,7168,0.0389141321182251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,8192,0.14206612904866536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,7168,0.12547199726104735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,7168,0.070796799659729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,6144,0.03392426570256551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,6144,0.11030506292978923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,5120,0.028860799471537274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,512,0.012898133198420206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,5120,0.09339520136515299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,5120,0.0523360013961792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,256,0.009822932879130046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,4096,0.02291626731554667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,3584,0.02049493392308553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,4096,0.0757589340209961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,3584,0.06478826602300009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,3584,0.03909333149592082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,3072,0.017821866273880004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,768,128,0.008413867155710856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,3072,0.05480320056279501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,2560,0.015618133544921874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,2560,0.04437333345413208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,2560,0.028630399703979494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,2048,0.013153066237767538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,2048,0.03601706822713216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,1536,0.010940800110499065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,1536,0.029845333099365233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,1536,0.01973653237024943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,1024,0.008584533135096233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,1024,0.02477226654688517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,65536,0.5846133550008138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,768,0.007406933108965556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,768,0.02262293299039205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,51200,0.4641855875651042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,512,0.006313600142796834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,512,0.020321067174275717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,512,0.011748266220092774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,256,0.005083733300367991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,256,0.018242132663726807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,8192,0.08002560138702393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,128,0.0042357335488001505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,128,0.01643519997596741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,6144,0.061884800593058266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,64,0.0039818666875362395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,64,0.016382933656374613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,512,32,0.004120533416668574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,512,32,0.016613333423932394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,4096,0.04325653314590454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,65536,0.1973152001698812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,3072,0.03424746592839559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,51200,0.1584789276123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,2048,0.02426346739133199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,65536,0.5218005180358887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,16384,0.055511466662089025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,51200,0.411513614654541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,65536,0.9645866394042969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,16384,0.25393706957499185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,12288,0.04374186595280965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,51200,0.7570346832275391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,12288,0.19433813095092772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,10240,0.036975999673207596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,768,0.013336533308029174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,10240,0.1643455982208252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,8192,0.02996480067571004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,8192,0.1343381404876709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,8192,0.07139413356781006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,256,0.009300266702969868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,7168,0.027485867341359455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,7168,0.11838613351186116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,6144,0.024133332570393882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,1024,0.015593600273132325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,6144,0.10273919900258381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,5120,0.01760106682777405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,512,128,0.008202666540940602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,5120,0.08809920152028403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,4096,0.0134634663661321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,16384,0.13765546480814617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,4096,0.03812053203582764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,3584,0.012129066387812297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,12288,0.10494933128356934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,3584,0.059157331784566246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,3072,0.01076586643854777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,3072,0.04902079900105794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,2560,0.009544533491134644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,10240,0.08814826806386312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,2560,0.04012586673100789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,2048,0.008241066833337148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,2048,0.03337493340174357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,2048,0.02091839909553528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,1536,0.0071381335457166035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,1536,0.02812906702359517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,1536,0.0172437330087026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,7168,0.06288319826126099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,1024,0.02364586591720581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,6144,0.05392640034357706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,768,0.005902933577696482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,768,0.021201066176096597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,4096,0.06971733570098877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,512,0.005277866621812185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,512,0.02028053402900696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,512,0.010251733660697936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,256,0.004251733422279358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,256,0.01728106737136841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,256,0.008059733112653096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,128,0.00377813329299291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,5120,0.046728531519571945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,128,0.016151466965675355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,128,0.006984533369541168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,64,0.0034453332424163817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,32,0.0035797332723935447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,64,0.015624533096949259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,256,32,0.016060800353686015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,65536,0.18316373825073243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,256,1024,0.006543999910354615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,51200,0.14674347241719563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,65536,0.503658676147461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,65536,0.9554592132568359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,16384,0.05234453280766806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,3584,0.03468480110168457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,51200,0.3976373354593913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,3072,0.029757867256800335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,12288,0.03629973332087199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,51200,0.7488159815470378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,16384,0.13425280253092448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,10240,0.030396799246470135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,16384,0.247322670618693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,12288,0.189464537302653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,2560,0.025106134017308553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,10240,0.16054506301879884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,7168,0.022401066621144612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,8192,0.069269331296285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,8192,0.1310634692509969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,7168,0.06090773344039917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,6144,0.020124799013137816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,7168,0.11499733130137127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,5120,0.01216319998105367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,6144,0.10023893515268963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,5120,0.08519999980926514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,4096,0.00963200032711029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,1024,0.013543466726938883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,4096,0.06687466303507486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,3584,0.00893440047899882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,4096,0.03659093379974365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,3584,0.03316160043080647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,3584,0.05671999851862589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,3072,0.008433066805203756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,256,768,0.011363200346628825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,2560,0.008926933010419209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,2560,0.037257599830627444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,2560,0.023450666666030885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,2048,0.007289599875609081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,2048,0.03204906582832336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,2048,0.019667200247446694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,8192,0.025115732351938886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,1536,0.006649599969387054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,1536,0.026909865935643512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,3072,0.04624533255894979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,1024,0.005474133292833964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,1024,0.0224671999613444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,12288,0.10307412942250568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,10240,0.08600640296936035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,768,0.004779733220736186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,768,0.020669867595036827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,512,0.004126933217048645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,512,0.018478933970133463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,6144,0.05279466708501181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,256,0.0035562666753927866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,256,0.017273600896199545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,5120,0.044674134254455565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,128,0.003252266595760981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,128,0.016114133596420287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,3072,0.027550933758417766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,64,0.0030752000709374744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,64,0.01564906636873881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,128,32,0.0031829332311948144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,128,32,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,65536,0.18035200436909993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,1536,0.016074666380882265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,51200,0.1424512068430583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,16384,0.051641599337259925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,16384,0.24896000226338705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,12288,0.038770135243733725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,65536,0.9481109619140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,10240,0.03201066652933757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,51200,0.7379082361857097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,12288,0.19029226303100585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,8192,0.027848533789316815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,7168,0.024634667237599692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,10240,0.15827840169270832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,6144,0.021563732624053956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,7168,0.11441600322723389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,6144,0.09881386756896973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,5120,0.011130666732788086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,4096,0.008796800176302593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,5120,0.0824682633082072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,4096,0.06555306514104207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,3584,0.008054399987061818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,3072,0.00823466678460439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,3584,0.05482133229573568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,768,0.010379733641942342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,3072,0.045076266924540205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,2560,0.007156266768773396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,2048,0.006055466830730438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,2560,0.03585813442866008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,1536,0.0054624001185099285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,2048,0.030747733513514203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,1024,0.0046015997727712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,512,0.00925546685854594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,768,0.004305066665013632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,1024,0.022447999318440756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,512,0.0037162666519482933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,768,0.01970133384068807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,256,0.0033130665620168054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,512,0.01890666683514913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,256,0.007271466652552287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,256,0.016289066274960837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,64,0.0030378667016824085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,128,0.015864533185958863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,32,0.0029887999097506206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,64,0.015060266852378846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,32,0.01551253298918406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,128,0.006344533463319142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,65536,0.17297706604003907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,51200,0.13676800727844238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3584,128,1024,0.012310399611790975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,16384,0.04554666678110759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,12288,0.03653759956359863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,10240,0.030723200241724653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,8192,0.12929386297861736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,8192,0.0256330668926239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,16384,0.24471999804178873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,10240,0.1595413366953532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,7168,0.023082667589187623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,8192,0.12880319754282635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,6144,0.020172800620396933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,5120,0.010735999544461567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,7168,0.11360320250193279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,4096,0.00829013337691625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,51200,0.7407306671142578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,5120,0.08278186321258545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,3584,0.007830399771531422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,4096,0.0650762677192688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,3072,0.0070826664566993715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,3584,0.05423466761906942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,2560,0.00633493314186732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,3072,0.044682665665944414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,2048,0.005558399856090546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,1536,0.004985600213209788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,2048,0.0303872009118398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,64,1536,0.026654932896296186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,1024,0.004199466605981191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,1536,0.02613653341929118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,768,0.0038805333276589714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,1024,0.02195626695950826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,768,0.020011732975641884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,64,128,0.00315733328461647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,256,0.0032426667710145317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,512,0.01816213329633077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,128,0.002899199972550074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,256,0.016401066382726034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,64,0.0028223998844623564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,128,0.015432533621788026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,32,0.0027903998891512555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,64,0.014939733346303306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,32,0.015085867047309876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,12288,0.18739627202351888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,6144,0.09860693613688151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,65536,0.9441685358683267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3584,32,2560,0.035946667194366455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3584,32,512,0.0034847999612490333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,16384,5.153223673502604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,12288,3.99224853515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,12288,7.1972503662109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,16384,7.705953979492188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,16384,9.597479248046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,12288,5.156802368164063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,10240,3.4756991068522134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,10240,4.540561930338542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,10240,6.017913818359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,8192,3.2324373881022135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,8192,5.881018575032552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,7168,2.535643768310547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,8192,4.143098704020182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,51200,17.43675333658854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,7168,3.0775126139322917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,7168,4.873476155598959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,6144,2.3665120442708334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,6144,2.580973815917969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,6144,3.7463595072428384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,5120,1.6727391560872396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,4096,1.3147744496663412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,5120,2.16213862101237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,5120,3.1226656595865885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,4096,2.559320576985677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,3584,1.2271925608317056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,3584,2.1438357035319013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,3584,1.5267232259114585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,3072,0.9925098419189453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,3072,1.8667840321858722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,3072,1.3293963114420573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,2560,0.884610112508138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,2560,1.118561045328776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,2560,1.535954157511393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,2048,0.6823765436808269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,2048,1.2723988850911458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,2048,0.9579455693562826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,1536,0.5461535771687825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,1536,0.9198762893676757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,1536,0.7835701624552409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,1024,0.4055103937784831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,4096,1.6971691131591797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,1024,0.6616138458251953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,1024,0.5926698684692383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,768,0.3248682657877604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,768,0.5180469195048014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,512,0.24513707160949708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,512,0.3896544138590495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,256,0.20475200017293296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,768,0.546774419148763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,256,0.19866453806559245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,128,0.1841759999593099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,512,0.41938133239746095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,256,0.3668277422587077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,128,0.18487787246704102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,64,0.17239999771118164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,64,0.181932799021403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,32,0.16748266220092772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,128,0.3198368072509766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,65536,32,0.17932586669921874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,65536,51200,31.275636800130208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,65536,51200,33.27879028320312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,16384,4.1100318908691404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,16384,7.655878194173177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,51200,13.502813720703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,16384,5.768464152018229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,65536,16.813636271158853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,12288,3.4125493367513022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,12288,6.304578145345053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,10240,2.8453961690266927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,12288,4.460552469889323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,10240,5.0333714803059895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,10240,3.5692649841308595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,8192,2.147886912027995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,51200,24.33367919921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,51200,24.5688720703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,8192,3.9728266398111978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,8192,2.818869272867839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,7168,1.9175093332926434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,7168,2.3660181681315104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,7168,3.3225588480631507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,6144,1.6696895599365233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,6144,2.9317291259765623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,5120,1.3221579233805338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,5120,2.389914703369141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,4096,1.0747594197591146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,4096,2.0861066182454424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,4096,1.3736661275227866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,3584,1.7998687744140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,3584,0.9944298426310221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,3584,1.2016469319661458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,6144,1.967843246459961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,3072,0.8932928085327149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,65536,31.47217610677083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,3072,1.5499168395996095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,5120,1.5075499216715493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,3072,1.1112373352050782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,2560,0.7386485417683919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,2560,1.1976020812988282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,2048,0.593553098042806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,2048,1.0671263376871745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,1536,0.4448853174845378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,2048,0.7816832224527995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,1536,0.7567850748697917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,65536,33.30227457682292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,1024,0.47364053726196287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,1024,0.3432554562886556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,768,0.37847146987915037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,1024,0.4709514617919922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,768,0.2852853457132975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,512,0.2721877415974935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,512,0.2013024012247721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,256,0.17007039388020834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,512,0.33438612620035807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,768,0.4038005193074544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,128,0.15308267275492352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,128,0.15715413093566893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,128,0.24331092834472656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,64,0.16207787195841472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,64,0.15364586512247722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,32,0.1391647974650065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,51200,32,0.14893546104431152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,51200,256,0.18693013191223146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,2560,0.8332768122355143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,1536,0.5524319966634115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,51200,256,0.25880319277445474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,16384,1.361353556315104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,16384,2.510419209798177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,51200,4.436808776855469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,65536,5.869218953450521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,16384,1.7257077534993488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,12288,1.7791968027750653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,12288,1.0246325174967448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,12288,1.2578261057535807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,10240,0.8882026672363281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,51200,7.737508138020833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,10240,1.5556949615478515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,51200,6.742153422037761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,8192,0.705674680074056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,65536,9.286201985677083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,8192,1.2588981628417968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,7168,0.6161141077677409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,8192,0.891547711690267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,7168,1.043555196126302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,7168,0.7701205571492513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,6144,0.548533312479655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,5120,0.46734933853149413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,6144,0.6275583902994792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,65536,10.13670145670573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,6144,0.9324458440144857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,5120,0.7989290873209636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,4096,0.3669823964436849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,4096,0.4441472053527832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,4096,0.6097514470418294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,3584,0.3452661196390788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,3584,0.42261546452840165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,3584,0.5447861353556316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,3072,0.43672959009806317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,2560,0.3744160016377767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,2560,0.25704107284545896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,2048,0.30084158579508463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,2560,0.30338986714680993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,2048,0.22596373558044433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,2048,0.2533173402150472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,1536,0.16917333602905274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,1024,0.16017386118570964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,1024,0.12356692949930828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,768,0.12389226754506429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,1024,0.1620202700297038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,768,0.10103360017140706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,768,0.14399466514587403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,512,0.09201920032501221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,512,0.07690026760101318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,256,0.05676800012588501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,512,0.1153450647989909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,3072,0.2964106559753418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,256,0.06578666766484578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,256,0.09421439965566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,128,0.050600532690684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,128,0.06046293179194132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,3072,0.37171414693196614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,64,0.0460042675336202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,128,0.08549119631449381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,32,0.04673600196838379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,64,0.06019413471221924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,16384,32,0.06059626738230387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,16384,1536,0.24479146003723146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,10240,1.048359489440918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,5120,0.516158930460612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,16384,1536,0.19701120058695476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,51200,3.3582400004069006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,16384,1.8260949452718098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,65536,4.3059443155924475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,16384,1.0726324717203775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,51200,4.162834167480469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,16384,1.298040517171224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,12288,0.8055871963500977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,12288,1.3446943918863932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,12288,0.9514453252156576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,51200,5.503399658203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,10240,1.094283676147461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,10240,0.6875466664632162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,8192,0.5720629374186198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,8192,0.897707748413086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,7168,0.494158935546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,7168,0.7733952204386394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,65536,7.092582194010417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,6144,0.6733376185099285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,6144,0.429200013478597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,65536,6.3888605753580725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,5120,0.577230962117513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,5120,0.3821621259053548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,5120,0.4144010543823242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,4096,0.2929984092712402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,4096,0.41916694641113283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,3584,0.38703254063924153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,3584,0.2537951946258545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,3584,0.300981330871582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,3072,0.3413344065348307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,3072,0.21976426442464195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,3072,0.2622709274291992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,2560,0.18640853563944498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,2560,0.2884895960489909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,2560,0.22740586598714194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,2048,0.21487040519714357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,2048,0.16850773493448895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,1536,0.1652703921000163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,1536,0.13097279866536457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,1024,0.11751893361409504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,10240,0.793010139465332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,1536,0.16203200022379557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,768,0.0944373369216919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,8192,0.5984959920247396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,768,0.0842570702234904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,512,0.07192959785461425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,768,0.09971946875254313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,512,0.06388479868570963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,512,0.08332160313924154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,256,0.045414400100708005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,7168,0.5400437037150065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,256,0.05469013452529907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,128,0.04038613239924113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,256,0.07073706785837809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,128,0.049060265223185226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,64,0.0356170654296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,12288,32,0.03622399965922038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,64,0.05033493439356486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,32,0.049975466728210446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,6144,0.4609535853068034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,12288,1024,0.09680213133494059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,4096,0.3236768086751302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,2048,0.1783594608306885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,1024,0.11361599763234456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,12288,128,0.06421440045038859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,16384,1.5497056325276692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,51200,3.110667673746745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,65536,4.032466125488281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,16384,0.9451701482137045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,12288,0.7186293284098307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,12288,0.8056608200073242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,16384,1.098257064819336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,65536,5.08242441813151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,12288,1.188777542114258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,51200,4.759732055664062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,51200,3.55721181233724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,10240,0.6379402796427409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,8192,0.5238176027933756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,65536,5.936807250976562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,10240,0.9563392003377279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,8192,0.7608800252278646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,8192,0.5463551839192708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,7168,0.43459199269612625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,7168,0.6904501597086589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,7168,0.4913130760192871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,6144,0.40004479090372724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,6144,0.4137440045674642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,5120,0.4678741455078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,6144,0.5841450373331706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,5120,0.31595306396484374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,4096,0.2742666562398275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,5120,0.3592597325642904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,4096,0.4011359850565593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,4096,0.30076373418172203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,3584,0.24461867014567057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,3584,0.3288053194681803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,3072,0.2762997309366862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,3072,0.21340479850769042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,2560,0.1784554640452067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,2560,0.24014080365498863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,2560,0.20370987256368003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,2048,0.19772693316141765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,2048,0.15233813921610515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,2048,0.1737738609313965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,1536,0.15797653198242187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,1536,0.12090240319569905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,1024,0.10457066694895427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,1536,0.1345802625020345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,1024,0.08917760054270427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,1024,0.11100906531016033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,768,0.0739466667175293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,512,0.062388265132904054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,768,0.09137813250223795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,512,0.05818666617075602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,256,0.03991466760635376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,256,0.04907840092976888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,128,0.03399466673533122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,128,0.04491946697235107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,128,0.0563754677772522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,64,0.03065599997838338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,64,0.04534293413162231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,32,0.03139306704203288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,10240,32,0.044609065850575766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,3072,0.2561994711558024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,10240,0.6624842961629231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,10240,768,0.08390613396962485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,3584,0.24457707405090331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,512,0.07235199610392253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,10240,256,0.060078934828440345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,65536,3.3443222045898438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,16384,0.8385567982991537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,16384,1.2168139139811198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,51200,2.5389686584472657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,65536,3.905416615804037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,51200,3.9982304890950524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,12288,0.6814773559570313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,12288,0.9668928146362304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,65536,5.066663614908854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,12288,0.6616170883178711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,10240,0.5697418848673503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,10240,0.5354293187459309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,10240,0.7820063908894856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,8192,0.6585919698079427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,7168,0.37237974802652996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,8192,0.4601258595784505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,7168,0.5555253346761068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,6144,0.4877663930257161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,6144,0.32282028198242185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,6144,0.3596949259440104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,5120,0.4148415883382161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,5120,0.2781503995259603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,4096,0.3120352109273275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,51200,2.8579925537109374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,4096,0.23822612762451173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,3584,0.2853109359741211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,3584,0.22980480194091796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,3584,0.19230613708496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,3072,0.24832746187845864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,3072,0.17714667320251465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,2560,0.21236054102579752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,2560,0.1838581403096517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,2048,0.16890239715576172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,2048,0.13029759724934895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,2048,0.13857280413309733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,1536,0.1200287977854411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,1536,0.09926079909006755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,3072,0.17596373558044434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,1536,0.10924159685770671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,1024,0.08400426705678304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,1024,0.07535466353098551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,2560,0.14452479680379232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,1024,0.08269973595937094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,768,0.06598506768544515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,8192,0.4392650604248047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,768,0.06306560039520263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,512,0.04922240177790324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,768,0.07246932983398438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,512,0.055777064959208165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,256,0.03323946595191955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,512,0.06085120042165121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,128,0.02755733331044515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,7168,0.3848896026611328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,256,0.05575466553370158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,128,0.03973013162612915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,16384,0.8825504302978515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,32,0.02629759907722473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,32,0.04030613501866658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,5120,0.27648534774780276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,4096,0.2292405287424723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,256,0.042208000024159746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,8192,64,0.02573653260866801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,8192,64,0.040354132652282715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,16384,0.9678496042887369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,65536,2.9616864522298174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,8192,128,0.04685973326365153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,16384,0.7558762868245442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,65536,3.1649449666341143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,51200,2.25972900390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,51200,2.540940856933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,12288,0.5429354349772135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,12288,0.7284181594848633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,16384,0.7911466598510742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,51200,3.4180661519368494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,65536,4.258983357747396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,10240,0.46370665232340497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,10240,0.667079480489095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,8192,0.5501045227050781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,8192,0.3705322583516439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,8192,0.4214474678039551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,7168,0.43849814732869463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,7168,0.3478325208028158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,6144,0.3637162526448568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,6144,0.3047306696573893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,6144,0.32188374201456704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,5120,0.25598506927490233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,5120,0.31898771921793617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,5120,0.2663573265075684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,4096,0.26399146715799965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,4096,0.21490559577941895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,4096,0.2250666618347168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,3584,0.22005440394083658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,3584,0.18321599960327148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,3072,0.19568746884663898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,3584,0.1939359982808431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,3072,0.1567530632019043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,3072,0.17477866808573406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,2560,0.1638869285583496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,2560,0.1314687967300415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,2048,0.13112959861755372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,2048,0.12253013451894124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,2048,0.11769706408182781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,1536,0.1049162705739339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,1024,0.07563947041829427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,1536,0.09237013657887777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,7168,0.3740490595499674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,1024,0.072161070505778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,1024,0.076583464940389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,768,0.057468799750010166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,768,0.060439467430114746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,512,0.04320426781972249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,512,0.04861653248469035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,256,0.030446932713190718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,256,0.03961173295974731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,256,0.046325333913167316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,128,0.025433599948883057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,128,0.03679253260294597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,12288,0.5661951700846355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,128,0.04304639895757039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,64,0.02296853264172872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,7168,32,0.02377706567446391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,10240,0.47321386337280275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,64,0.03694933255513509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,7168,32,0.03810880184173584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,512,0.05493866602579752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,2560,0.1360106627146403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,1536,0.0952842632929484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,7168,768,0.06562240123748779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,65536,2.4965194702148437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,51200,1.9972522735595704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,16384,0.8258261362711588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,65536,2.879015350341797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,51200,2.841602071126302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,16384,0.6221141179402669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,16384,0.6540063858032227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,65536,3.4213312784830725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,12288,0.5130549430847168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,10240,0.41068480809529617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,12288,0.6685141245524089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,10240,0.5278090794881185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,12288,0.526142946879069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,8192,0.3371765454610189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,51200,2.1284149169921873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,10240,0.43438186645507815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,8192,0.3361845334370931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,8192,0.4387978553771973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,7168,0.4022709210713704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,6144,0.3102677345275879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,5120,0.22665279706319175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,6144,0.27653439839680993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,6144,0.28800106048583984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,5120,0.27795626322428385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,5120,0.2395690600077311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,4096,0.1791744073232015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,4096,0.19055360158284504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,3584,0.16308266321818035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,3072,0.16282666524251302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,3072,0.14284906387329102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,3072,0.14812266031901042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,2560,0.1383786678314209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,2560,0.12071999708811443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,2048,0.1139946699142456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,2048,0.10016746520996093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,2048,0.10702719688415527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,1536,0.08642133076985678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,1536,0.08200213114420572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,1024,0.06356586615244547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,7168,0.3024885177612305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,1024,0.06204479932785034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,1024,0.06378879944483438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,768,0.048978133996327715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,768,0.051595731576283776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,4096,0.22953492800394693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,3584,0.19740479787190754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,3584,0.17203200658162435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,512,0.0378389318784078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,256,0.026014933983484905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,512,0.044649600982666016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,512,0.04714346726735433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,128,0.021606399615605672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,256,0.0360042651494344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,128,0.0357312003771464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,64,0.01890986760457357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,64,0.03309119939804077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,6144,32,0.020071466763814293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,32,0.03386559883753459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,7168,0.28707412083943684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,6144,128,0.03312106728553772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,2560,0.11713706652323405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,1536,0.08037973244984944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,768,0.05523733297983805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,6144,256,0.038203732172648115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,65536,2.304617563883464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,65536,3.186573791503906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,16384,0.6969866434733073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,16384,0.5708576202392578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,12288,0.5446677525838216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,51200,1.7762388865152996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,12288,0.46512854894002276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,51200,2.537114715576172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,10240,0.47481387456258134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,10240,0.38106346130371094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,8192,0.4054527918497722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,8192,0.320030943552653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,7168,0.32464640935262046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,7168,0.2708330790201823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,6144,0.2726762771606445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,6144,0.23740480740865072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,16384,0.7266421635945638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,12288,0.5346879959106445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,5120,0.23632319768269858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,5120,0.2070847988128662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,10240,0.45266345342000325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,4096,0.1896224021911621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,4096,0.16179413795471193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,3584,0.16014933586120605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,65536,3.4484074910481772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,8192,0.3648597399393717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,51200,2.5814219156901044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,3584,0.1409013271331787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,3072,0.14172479311625164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,3072,0.13735466003417968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,2560,0.11952746709187825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,2560,0.11495467027028401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,7168,0.3245386759440104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,2048,0.09692160288492838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,2048,0.08884479999542236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,1536,0.07449386914571127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,1536,0.07328426837921143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,6144,0.2675679842631022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,1024,0.05324480136235556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,1024,0.056866133213043214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,1024,0.05639359951019287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,768,0.04269760052363078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,768,0.048767999807993574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,5120,0.22255360285441078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,512,0.03284693360328674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,512,0.041790934403737386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,4096,0.18223360379536946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,256,0.022746666272481283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,256,0.03210879961649577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,3584,0.1596992015838623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,3072,0.13446079889933268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,128,0.017522132396697997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,128,0.029623466730117797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,64,0.015465600291887918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,2560,0.11865706443786621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,64,0.029252266883850096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,5120,32,0.01641386648019155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,5120,32,0.02985493342081706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,2048,0.0960256020228068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,1536,0.07208213011423746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,768,0.04440639813741048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,65536,1.955191421508789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,65536,2.6032681783040363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,51200,1.9801024119059245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,512,0.0360213319460551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,51200,1.5473930358886718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,256,0.029355732599894206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,5120,128,0.02485760052998861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,16384,0.5834837595621745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,16384,0.5091391881306966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,12288,0.44646933873494465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,12288,0.3790250778198242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,12288,0.355018679300944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,10240,0.32414185206095375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,10240,0.38296000162760413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,10240,0.3211327870686849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,8192,0.2875338554382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,51200,1.540996297200521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,8192,0.27787307103474934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,7168,0.26653760274251304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,7168,0.22986240386962892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,7168,0.24838186899820963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,6144,0.22314346631368004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,5120,0.20013653437296547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,5120,0.18622934023539225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,4096,0.15449706713358563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,4096,0.14815786679585774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,65536,2.0079104105631513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,3584,0.15280960400899252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,6144,0.21480533281962075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,3584,0.13966399828592937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,3584,0.114956800142924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,3072,0.12629226843516034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,3072,0.108570663134257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,2560,0.09686506589253743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,2560,0.09415573279062907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,16384,0.4644501368204753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,2560,0.08737493356068929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,2048,0.08387306531270346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,2048,0.08049706617991129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,2048,0.0811562697092692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,1536,0.061604265371958414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,1536,0.07234133084615071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,1024,0.045655465126037596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,1024,0.05329493284225464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,8192,0.23894186019897462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,768,0.036510932445526126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,768,0.04480640093485515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,512,0.028759467601776122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,768,0.04158399899800618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,512,0.03510186672210693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,512,0.03781546751658122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,256,0.020222934087117513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,256,0.0303872009118398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,128,0.015468800067901611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,128,0.02751893401145935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,4096,0.13053226470947266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,128,0.025975465774536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,64,0.013688533504803976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,64,0.02707306742668152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,4096,32,0.014088533322016396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,4096,32,0.02704533338546753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,6144,0.18383679389953614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,5120,0.15568960507710775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,3072,0.09994346300760905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,1536,0.05990293423334757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,65536,1.7943626403808595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,1024,0.04715840021769206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,51200,1.576919428507487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,16384,0.5001472155253093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,51200,1.3953589121500651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,65536,1.80328852335612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,16384,0.4573674519856771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,12288,0.40534079869588213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,12288,0.3581930796305338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,4096,256,0.028521599372227986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,16384,0.4797877311706543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,10240,0.3059178670247396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,8192,0.2434336026509603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,10240,0.32509120305379235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,8192,0.26553279558817544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,8192,0.2279754638671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,51200,1.3011978149414063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,7168,0.21451199849446617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,6144,0.18734505971272786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,7168,0.22325013478597006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,7168,0.20196053187052407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,6144,0.20149653752644858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,65536,2.211835734049479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,5120,0.15784427324930828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,5120,0.1464896043141683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,5120,0.16024746894836425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,4096,0.1365898609161377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,4096,0.12674132982889813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,3584,0.11363306840260823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,3584,0.11093653043111165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,3072,0.09837973117828369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,3072,0.09987839857737223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,2560,0.08315093517303467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,2560,0.08677439689636231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,2048,0.066920534769694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,2048,0.07418879667917887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,2048,0.06733973026275634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,1536,0.05218133529027304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,1536,0.061178668340047204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,1536,0.05490560134251913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,1024,0.03777066469192505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,1024,0.04752853314081828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,3584,0.11267093022664387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,768,0.031032532453536987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,768,0.04014613231023152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,12288,0.31761598587036133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,512,0.024284799893697105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,10240,0.26609919865926107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,512,0.03141760031382243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,512,0.03438933293024699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,256,0.017173333962758382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,256,0.027853866418202717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,256,0.02600319981575012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,128,0.013168000181516013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,128,0.026101332902908326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,6144,0.16645013491312663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,128,0.023613866170247397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,64,0.011539199948310852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3584,32,0.012000000476837159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,64,0.025745066006978352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,4096,0.11594666639963787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3584,32,0.02526719967524211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,3072,0.09103146394093832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,2560,0.07936426798502604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,1024,0.04256639877955119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,51200,1.3346666971842447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,65536,1.6580874125162761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,65536,1.613100814819336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,51200,1.286370086669922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3584,768,0.03732906579971314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,16384,0.4411338806152344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,16384,0.4482666651407878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,16384,0.3719200134277344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,12288,0.3131637255350749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,10240,0.2585045337677002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,10240,0.2338698705037435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,10240,0.2724639892578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,8192,0.20713386535644532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,51200,1.1966859181722005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,8192,0.21336426734924316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,8192,0.19147413571675617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,7168,0.19071680704752605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,6144,0.15496106147766114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,7168,0.2174506664276123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,6144,0.16473280588785807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,5120,0.1299189329147339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,5120,0.15862080256144206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,4096,0.1054965337117513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,4096,0.13252159754435222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,4096,0.10886720021565754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,3584,0.0977130651473999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,3584,0.10646506945292156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,12288,0.3085066795349121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,3072,0.08297066688537598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,3072,0.09145066738128663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,3072,0.08429866631825765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,2560,0.06878506342569987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,2560,0.0806389331817627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,2048,0.055956268310546876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,2048,0.0676736036936442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,2048,0.05956693490346273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,1536,0.0434986670811971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,7168,0.16700159708658854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,1536,0.05376960039138794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,1536,0.048197333017985025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,1024,0.03141973416010539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,1024,0.04083840052286784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,5120,0.12391040325164795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,768,0.025148799022038777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,768,0.03275199929873149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,768,0.034739200274149576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,512,0.019336533546447755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,65536,1.5437610626220704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,512,0.02940479914347331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,256,0.015711999932924905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,256,0.024606933196385704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,128,0.010102400183677673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,128,0.021948800484339396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,128,0.01984213391939799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,64,0.008674133817354839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,64,0.022115200757980347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,3072,32,0.00899839997291565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,3072,32,0.021922133366266885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,12288,0.2814688046773275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,6144,0.14651840527852375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,3584,0.09137919743855795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,65536,1.558782958984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,2560,0.06935359636942545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,65536,1.508722178141276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,16384,0.3516949335734049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,1024,0.037718399365743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,51200,1.2071295420328776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,512,0.026399999856948853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,16384,0.3979882558186849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,3072,256,0.022130133708318074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,51200,1.1877642313639323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,12288,0.2997653325398763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,12288,0.29329814910888674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,10240,0.22039999961853027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,10240,0.2510431925455729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,8192,0.1762272040049235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,8192,0.2030720074971517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,7168,0.15534292856852214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,7168,0.17928320566813152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,7168,0.1819210688273112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,6144,0.13509546915690104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,6144,0.15822933514912924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,5120,0.11268053054809571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,5120,0.1329866647720337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,65536,1.674722162882487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,16384,0.4032298723856608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,4096,0.0932032028834025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,12288,0.3089930534362793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,4096,0.11296532948811848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,10240,0.2550154685974121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,3584,0.08388480345408121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,3584,0.09589227040608725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,8192,0.20927039782206217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,3072,0.07105706532796224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,3072,0.08524479866027831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,2560,0.059824001789093015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,2560,0.07373013496398925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,6144,0.15078506469726563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,2048,0.048750933011372885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,2048,0.061110401153564455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,5120,0.12980480194091798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,1536,0.03796799977620442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,51200,1.3104938507080077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,1536,0.04951680103937785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,1024,0.02760106722513835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,1024,0.03778666655222575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,768,0.02250880002975464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,4096,0.1075551986694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,768,0.03222080071767171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,512,0.017412267128626504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,512,0.0292138675848643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,3584,0.09372586409250895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,256,0.012200533350308736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,3072,0.07922133604685465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,256,0.023622399568557738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,128,0.009574400385220845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,256,0.0166485329469045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,128,0.021553067366282146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,64,0.008155733346939087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,64,0.02177493373552958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2560,32,0.008524800340334576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,2560,0.06927573680877686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2560,32,0.021409066518147786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,2048,0.05533973375956217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,1536,0.04395519892374675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,1024,0.034782934188842776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,65536,1.2429300944010415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,65536,1.370901362101237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,51200,0.9496618906656901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,16384,0.2870378812154134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,51200,1.065010134379069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,16384,0.38741121292114256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,12288,0.21050559679667152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,768,0.026342399915059406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,12288,0.27769174575805666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,10240,0.1790826638539632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,10240,0.23411946296691893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,10240,0.2161344051361084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,128,0.014127999544143677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,8192,0.14717547098795575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,8192,0.18427093823750812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,7168,0.12924799919128419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,7168,0.16236693064371746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,6144,0.10965226491292317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,6144,0.14107200304667156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2560,512,0.02145599921544393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,5120,0.09072960217793782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,5120,0.1194048007329305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,5120,0.10841066837310791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,4096,0.07379093170166015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,4096,0.09709973335266113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,65536,1.410321044921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,3584,0.07036373615264893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,3584,0.09605226516723633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,16384,0.3342069307963053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,3584,0.0878976027170817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,51200,1.0675392150878906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,3072,0.05752000013987223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,2560,0.04849706490834554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,3072,0.0840885321299235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,12288,0.2584885279337565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,2560,0.06609066724777221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,2048,0.038839467366536456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,2048,0.047123201688130695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,2048,0.05541226863861084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,1536,0.03007359902064006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,1536,0.04306986729303996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,1024,0.02248426675796509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,1024,0.03399360179901123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,8192,0.17131840387980143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,768,0.01810986598332723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,768,0.02906879981358846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,768,0.022343466679255165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,7168,0.1528106689453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,512,0.013851733009020487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,512,0.025331199169158936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,256,0.010011733571688334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,256,0.022168533007303873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,6144,0.12762239774068196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,128,0.00793280005455017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,128,0.02027946710586548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,4096,0.0904970645904541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,64,0.006985599795977275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,64,0.020544000466664634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,2048,32,0.007397333284219106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,3072,0.06494293212890626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,2048,32,0.02023573319117228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,2560,0.05958400170008341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,1536,0.036541867256164554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,1024,0.030010666449864703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,65536,0.7817130406697591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,512,0.018332799275716148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,51200,0.6276373545328776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,65536,1.190988794962565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,256,0.014286933342615762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,16384,0.21653973261515297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,2048,128,0.012235732873280843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,16384,0.31160640716552734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,12288,0.16476160685221355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,51200,0.9520618438720703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,10240,0.1379637400309245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,12288,0.23742186228434242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,12288,0.22060160636901854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,10240,0.2020106633504232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,8192,0.10223999818166096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,8192,0.16257066726684571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,7168,0.09023786385854085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,7168,0.14486613273620605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,6144,0.07709546883900961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,6144,0.12661973635355633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,5120,0.06470186710357666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,5120,0.10815786520640056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,65536,1.1374869028727213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,4096,0.05496960083643595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,16384,0.2628714720408122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,4096,0.09277546405792236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,3584,0.04877653519312541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,51200,0.8915487925211588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,3584,0.0823263963063558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,3584,0.06855893135070801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,3072,0.04389013449350993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,3072,0.06951466401418051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,8192,0.13618027369181315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,2560,0.03572053511937459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,7168,0.12421332995096843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,2560,0.05732906659444174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,2048,0.028569600979487103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,2048,0.046930134296417236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,1536,0.022851200898488362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,1536,0.03791679938634236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,6144,0.10438613096872966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,1024,0.016860800981521606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,1024,0.029415466388066608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,10240,0.18112640380859374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,768,0.013915733496348063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,5120,0.08760746320088705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,768,0.026100265979766845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,512,0.01102186640103658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,512,0.02312320073445638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,4096,0.07214933236440023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,256,0.008289066453774769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,256,0.020270933707555137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,256,0.012122666835784912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,128,0.006913066903750102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,3072,0.05349973440170288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,128,0.018849066893259683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,64,0.006098133325576782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,64,0.01863573392232259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,2560,0.047814400990804036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1536,32,0.006348800162474315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1536,32,0.019245866934458414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,2048,0.03866879940032959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,1536,0.030107732613881426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,65536,0.5751466751098633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,1024,0.023848533630371094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,51200,0.43920427958170577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,16384,0.14648426373799642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,65536,1.0788501739501952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,768,0.018732800086339315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,16384,0.29836158752441405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,12288,0.10714240074157715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,512,0.015356799960136414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,51200,0.8974101384480795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,12288,0.22392640113830567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,10240,0.09091520309448242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,8192,0.07360426584879556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,10240,0.1860223929087321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,8192,0.14813973108927408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,7168,0.06480960051218668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,7168,0.13148799737294514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1536,128,0.010521599650382995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,6144,0.05448106527328491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,6144,0.11490560372670491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,65536,0.9394847869873046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,5120,0.04723306496938069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,5120,0.09827946821848552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,16384,0.23299199740091958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,12288,0.17482773462931317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,4096,0.038114134470621744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,51200,0.7236117045084636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,4096,0.07854506969451905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,3584,0.03347733418146769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,3584,0.06822400093078614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,10240,0.1512704054514567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,3072,0.02885119915008545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,3072,0.05928320089975993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,8192,0.12545493443806965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,2560,0.02537706693013509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,3072,0.04912000099817912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,2560,0.049459199110666915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,2048,0.021167999505996703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,2048,0.040941866238911946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,7168,0.1073749303817749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,1536,0.016531200210253397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,1536,0.03282346725463867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,6144,0.09116266568501791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,1024,0.012197333574295043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,1024,0.027693865696589152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,5120,0.07789546648661295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,768,0.010403199990590414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,4096,0.06424320141474406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,768,0.02435520092646281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,512,0.008459732929865519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,512,0.02134400010108948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,3584,0.058083200454711915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,256,0.006584533552328746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,256,0.019099734226862588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,2560,0.04191679954528808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,128,0.005636266867319743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,128,0.0180021325747172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,128,0.00920960009098053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,64,0.005075199902057648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,1024,32,0.005416533350944519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,2048,0.033930667241414386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,64,0.01800533334414164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,1024,32,0.01765973369280497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,1536,0.027447466055552167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,65536,0.42403628031412766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,51200,0.3300309181213379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,65536,0.8193034489949544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,65536,1.003329086303711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,16384,0.10581119855244954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,1024,0.020116267601648967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,51200,0.8001205444335937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,16384,0.259822940826416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,12288,0.0790986696879069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,10240,0.0664576013882955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,12288,0.20148372650146484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,10240,0.16786452929178874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,8192,0.05363626480102539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,8192,0.1380714734395345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,768,0.01699519952138265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,7168,0.0482805331548055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,512,0.013796266913414002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,7168,0.12339093685150146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,6144,0.041788800557454424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,1024,256,0.010955733060836793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,6144,0.10794026851654052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,5120,0.03552213509877523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,5120,0.09153813521067301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,5120,0.07039466698964437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,4096,0.02893120050430298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,4096,0.07362666924794516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,4096,0.057003732522328696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,3584,0.025563732782999678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,3584,0.0630240003267924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,51200,0.6474485397338867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,16384,0.2138144016265869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,3072,0.02216106653213501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,3072,0.05451093514760336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,12288,0.16241386731465657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,2560,0.019011199474334717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,2560,0.044709332784016925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,2048,0.01586026648680369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,10240,0.13846826553344727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,2048,0.03702186743418376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,1536,0.012719999750455221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,1536,0.029177600145339967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,8192,0.11030293305714925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,1024,0.009778133034706116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,1024,0.025117866198221844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,7168,0.09724906285603842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,768,0.008473599950472515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,768,0.022478934129079184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,6144,0.08409706751505533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,512,0.00697813332080841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,512,0.020424532890319824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,3584,0.050197335084279385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,256,0.005678933362166087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,256,0.01802240014076233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,3072,0.04348586797714234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,128,0.00492799977461497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,2560,0.03705919981002807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,128,0.017573332786560057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,64,0.004470400015513102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,64,0.016753067572911583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,2048,0.02990079919497172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,768,32,0.004690133531888326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,768,32,0.017850667238235474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,1536,0.023641600211461385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,65536,0.2889237403869629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,1024,0.017950934171676636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,51200,0.23215893109639488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,65536,0.5532800038655599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,16384,0.07747306823730468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,65536,0.9494496027628581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,51200,0.7470656077067057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,16384,0.2459008057912191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,16384,0.1442581335703532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,512,0.012602667013804117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,12288,0.05982933441797892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,10240,0.05032000144322714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,12288,0.11135573387145996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,12288,0.18757012685139973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,8192,0.03956480026245117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,10240,0.16113492647806804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,8192,0.13128746350606282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,7168,0.035164801279703777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,256,0.009774933258692425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,7168,0.11661653518676758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,6144,0.030502400795618695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,7168,0.06695893605550131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,6144,0.058616534868876136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,6144,0.10160319805145264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,768,0.015324800213177999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,5120,0.02619413336118062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,5120,0.08578773339589438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,4096,0.020881066719690956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,5120,0.05032106637954712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,4096,0.0688373327255249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,3584,0.018832000096638997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,768,128,0.008334933718045553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,3584,0.058658134937286374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,3584,0.03760639826456706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,3072,0.016707199811935424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,2560,0.014371200402577718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,3072,0.049371735254923506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,2560,0.040598400433858234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,2048,0.012150399883588155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,10240,0.09347626368204752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,2048,0.03439573446909587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,2048,0.022932267189025878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,1536,0.010110933581988018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,1536,0.027998934189478557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,1024,0.007911466558774312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,1024,0.024078933397928874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,1024,0.014936533570289613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,768,0.006958933174610138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,768,0.021399466196695964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,768,0.013091199596722922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,512,0.005774933099746704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,512,0.01934400002161662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,51200,0.4391669273376465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,512,0.011203199625015259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,256,0.00488319993019104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,256,0.018323200941085815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,128,0.004583466549714406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,128,0.008123733103275299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,128,0.01685439944267273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,64,0.004080000023047129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,512,32,0.004215466479460398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,64,0.016291200121243795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,512,32,0.016272000471750894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,65536,0.17768640518188478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,8192,0.0767029364903768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,51200,0.14261120160420734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,65536,0.8902687708536783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,3072,0.03253013292948405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,4096,0.041205334663391116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,51200,0.7011072158813476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,16384,0.05629013379414877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,12288,0.039867734909057616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,2560,0.027399466435114546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,16384,0.13113813400268554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,16384,0.23561813036600748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,10240,0.0341599980990092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,12288,0.09904747009277344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,12288,0.18065279324849445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,10240,0.08331200281778971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,10240,0.1532416025797526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,1536,0.019476266702016194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,7168,0.024846933285395303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,8192,0.06715733210245768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,8192,0.12528639634450275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,6144,0.021921066443125407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,7168,0.059005868434906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,7168,0.11034453709920247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,5120,0.01537493367989858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,6144,0.05179733435312907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,6144,0.09624959627787272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,4096,0.012684800227483115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,5120,0.044121599197387694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,5120,0.08001493612925212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,3584,0.011115733782450359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,4096,0.06231573422749838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,3072,0.010129066308339436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,3584,0.053345068295796716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,3072,0.04397653341293335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,2560,0.00918293297290802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,2560,0.03604053258895874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,512,256,0.009133866429328919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,2048,0.007874133189519246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,2048,0.03229333360989888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,2048,0.020067199071248373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,1536,0.006922666728496551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,8192,0.027346134185791016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,1536,0.026603732506434125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,1536,0.01699840029080709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,1024,0.0057205334305763245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,1024,0.0226197342077891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,65536,0.49276161193847656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,3584,0.032813866933186844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,768,0.0050901333491007485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,768,0.011142399907112122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,768,0.02036693294843038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,512,0.0045962666471799215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,512,0.009818666179974874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,512,0.018806399901707967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,256,0.0038762666285037995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,256,0.017548799514770508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,256,0.00811413327852885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,128,0.003730133424202601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,128,0.0162090669075648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,64,0.0034730667869249977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,64,0.015713066856066386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,256,32,0.003504000107447306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,51200,0.39773972829182946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,256,32,0.015706666310628257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,65536,0.17181226412455242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,4096,0.03634666601816813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,65536,0.877352523803711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,3072,0.02760106722513835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,51200,0.13873066902160644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,16384,0.04782506624857585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,2560,0.023966934283574423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,51200,0.3722954750061035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,16384,0.22900479634602866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,12288,0.03824640115102132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,16384,0.12592533429463704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,51200,0.6896778742472331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,10240,0.028964267174402876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,12288,0.09622399806976319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,12288,0.17787413597106932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,8192,0.023652267456054688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,10240,0.0803167978922526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,10240,0.15074879328409832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,7168,0.0211733341217041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,8192,0.12309226989746094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,7168,0.057341865698496496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,6144,0.017882666985193887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,7168,0.10812373161315918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,5120,0.011370666821797689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,6144,0.049621331691741946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,5120,0.07833600044250488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,4096,0.009875200192133586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,4096,0.06042773326237997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,3584,0.009077333410580953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,128,0.007055999835332234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,3584,0.050887465476989746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,3072,0.008036266764005024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,3072,0.04259093205134074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,3072,0.02587413390477498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,2560,0.007340799768765767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,2560,0.034550400575002034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,6144,0.09461013476053873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,2048,0.006585599978764851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,2048,0.03071039915084839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,256,1024,0.01291306714216868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,2048,0.0187882661819458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,1536,0.005717333157857259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,1536,0.026261333624521894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,65536,0.47583573659261064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,1024,0.005303466816743215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,1024,0.011923199892044068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,1024,0.021713066101074218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,768,0.004757333298524221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,768,0.01959999998410543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,512,0.00401706670721372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,512,0.01832533280054728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,512,0.008890666564305623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,256,0.0034901333351929987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,256,0.01663040022055308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,256,0.00721919983625412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,128,0.0032597333192825317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,128,0.015734400351842245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,128,0.006187733511130015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,64,0.0030858665704727173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,64,0.015082666277885437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,128,32,0.0031199999153614043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,8192,0.06513386567433675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,128,32,0.015020799636840821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,65536,0.16042453447977703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,5120,0.04224960009256999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,51200,0.1277120033899943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,1536,0.015918933351834617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,16384,0.04453440109888713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,12288,0.03496640125910441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,4096,0.0346560001373291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,16384,0.2313920021057129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,10240,0.03057919939359029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,12288,0.1753109296162923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,8192,0.02566933234532674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,10240,0.14879892667134603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,7168,0.020809600750605263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,8192,0.12112212975819905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,6144,0.014335999886194864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,51200,0.6827775955200195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,7168,0.10728320280710857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,5120,0.011528533697128297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,6144,0.09286613464355468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,4096,0.01018346647421519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,5120,0.07762347062428793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,3584,0.009290666381518046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,4096,0.05978133281071981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,3072,0.008265600105126699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,3584,0.04989226659138997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,2560,0.007776000102361043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,2048,0.0067893331249554946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,2560,0.03417173226674398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,2048,0.02946666677792867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,1536,0.025091199080149333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,1024,0.0046517332394917805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,1024,0.02245866656303406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,2560,0.022376533349355063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,768,0.0042133331298828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,512,0.0037920000652472176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,768,0.019896533091862997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,256,0.003420799970626831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,512,0.018633600076039633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,128,0.0031136001149813337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,256,0.01623040040334066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,64,0.002995199958483378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,128,0.015619200468063355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,64,0.015220266580581666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,32,0.0029685333371162414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,32,0.015609600146611533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,65536,0.16289706230163575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,768,0.010151466727256775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,65536,0.8730474472045898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,51200,0.1270645300547282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,16384,0.04681066672007243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,12288,0.03664960066477458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3329,128,3584,0.030935466289520264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,10240,0.02954346736272176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,12288,0.1764992078145345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,16384,0.2298133373260498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,64,3072,0.04124586582183838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,8192,0.02685760060946147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,7168,0.02197973330815633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,64,1536,0.005551999807357788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,6144,0.016876800855000814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,10240,0.1484992027282715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,5120,0.01218666632970174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,7168,0.10727573235829671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,4096,0.010240000486373902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,6144,0.0926805337270101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,51200,0.6886058807373047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,3584,0.008810666203498841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,65536,0.8785418828328451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,3072,0.007986133297284443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,2560,0.007266133526961009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,3584,0.049872001012166337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,2048,0.006404266754786174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,2560,0.03335040012995402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,1536,0.005122133096059163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,2048,0.02895786762237549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,1024,0.004385066529115042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,1536,0.02508373260498047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,768,0.004045866678158442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,1024,0.022066134214401244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,512,0.0036447999378045404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,768,0.01957119901974996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,256,0.0031541332602500914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,512,0.018076799313227334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,128,0.0028938665986061097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,256,0.016195199886957803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,64,0.0027829334139823914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,128,0.01532373329003652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,64,0.014980266491572062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,32,0.014859732985496522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,8192,0.12165226936340331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,5120,0.07598293622334798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,4096,0.06003733476003011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3329,32,3072,0.04079999923706055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3329,32,32,0.002842666705449422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,12288,3.906611124674479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,16384,5.248548380533854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,16384,6.116560872395834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,12288,7.3155970255533855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,16384,9.65538330078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,10240,3.2418955485026038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,10240,6.145393880208333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,10240,4.020185597737631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,12288,4.816105651855469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,8192,3.0392969767252604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,8192,5.933626810709635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,7168,2.321422831217448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,7168,2.9452255249023436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,51200,17.657039388020834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,7168,4.832322184244791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,6144,2.115826161702474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,8192,3.1850186665852864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,5120,1.6465151468912762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,6144,3.7482869466145834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,5120,3.0747060139973956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,4096,1.3636330922444662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,4096,2.4796511332194013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,4096,1.6237354278564453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,3584,1.1805290222167968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,6144,2.4931243896484374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,3584,2.1897310892740887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,5120,2.0463551839192706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,3072,1.0426783879597983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,3072,1.9094132741292316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,2560,0.8938410441080729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,2560,1.0569215774536134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,2560,1.577233123779297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,2048,0.7376981099446615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,2048,1.3277076721191405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,1536,1.045365333557129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,3584,1.4416095733642578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,1536,0.5593173344930013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,1024,0.4356714566548665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,1024,0.6870954513549805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,51200,23.080069986979165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,1024,0.5743616104125977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,768,0.3528277397155762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,768,0.5178240140279133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,3072,1.2570229848225911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,512,0.36334292093912757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,512,0.26200960477193197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,256,0.26041173934936523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,768,0.559874153137207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,256,0.19928639729817707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,128,0.1846453348795573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,128,0.19012160301208497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,256,0.32279465993245443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,64,0.17098453839619954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,128,0.30880107879638674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,32,0.16619092623392742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,64,0.1806378682454427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,65536,32,0.17842453320821125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,2048,0.8603253046671548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,1536,0.6873247782389323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,65536,512,0.37149120966593424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,65536,51200,31.918625895182295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,16384,4.332363891601562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,16384,7.562204996744792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,16384,4.862989807128907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,51200,13.179378255208334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,12288,6.458356221516927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,65536,17.42743937174479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,12288,3.33832753499349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,12288,3.8365099589029947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,51200,18.493609619140624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,10240,2.7676971435546873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,10240,3.16591796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,10240,4.85993398030599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,8192,2.1021344502766928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,8192,3.7663988749186195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,8192,2.647142283121745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,7168,1.824070358276367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,7168,2.13352533976237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,7168,3.2905258178710937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,51200,24.140899658203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,6144,1.6124331156412761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,6144,2.846566518147786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,6144,1.8238965352376302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,5120,1.3057013193766276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,65536,25.758721923828126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,5120,2.4487380981445312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,5120,1.6166815439860027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,4096,1.0347541173299155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,3584,0.9509077072143555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,3072,0.8689994812011719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,3584,1.682034174601237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,3584,1.1719317118326822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,3072,1.6358848571777345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,2560,0.6922186533610026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,2560,0.9259882609049479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,4096,2.1067850748697916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,2560,1.380246353149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,2048,0.5555306752522786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,2048,1.0858144124348958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,1536,0.493122132619222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,2048,0.778011703491211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,1536,0.7016789118448894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,4096,1.2257301330566406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,1024,0.5410261154174805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,1024,0.34347626368204753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,768,0.41237761179606125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,1024,0.49596694310506184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,768,0.3044917424519857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,512,0.29813013076782224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,512,0.22942934036254883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,768,0.4238613446553548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,256,0.19749760627746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,512,0.35669228235880535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,128,0.14673813184102374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,128,0.1499754587809245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,64,0.13545066515604656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,32,0.13344106674194336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,32,0.14448213577270508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,3072,0.9748661041259765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,51200,65536,31.957804361979168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,256,0.16286400159200032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,51200,64,0.14665493965148926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,1536,0.5326592127482097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,256,0.24949226379394532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,51200,128,0.23067946434020997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,16384,2.522459665934245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,51200,4.412132263183594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,65536,5.907868957519531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,16384,1.4357962290445963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,65536,6.863370768229167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,12288,1.0831136067708333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,51200,4.963280232747396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,12288,1.7958197275797527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,12288,1.2155391693115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,51200,7.552338155110677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,10240,0.9471018473307291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,8192,0.7659562428792317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,10240,1.5036960601806642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,65536,9.953762817382813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,7168,0.651414426167806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,7168,1.1369803110758463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,16384,1.632312520345052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,6144,0.5526549021402996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,10240,1.0208447774251304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,8192,1.2069087982177735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,6144,1.008680534362793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,5120,0.4709984143575032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,5120,0.48879254659016924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,5120,0.8055039723714194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,4096,0.5720501581827799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,4096,0.4095754623413086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,4096,0.40474348068237304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,3584,0.3523925463358561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,3584,0.35436585744222004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,3584,0.5424885431925456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,3072,0.28842665354410807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,3072,0.4638058662414551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,3072,0.3272639910380045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,2560,0.2639359951019287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,2560,0.3830975850423177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,2560,0.2799829483032227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,2048,0.20380585988362632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,2048,0.25563626289367675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,1536,0.2537450631459554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,2048,0.3320063908894857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,1536,0.1653066635131836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,1024,0.16036052703857423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,1536,0.19463253021240234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,8192,0.7869792302449544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,1024,0.1367253303527832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,768,0.12368960380554199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,512,0.09012586275736491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,512,0.09271893501281739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,512,0.10876479943593342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,256,0.05637866655985514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,128,0.049586133162180586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,256,0.06913599967956544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,128,0.06169066826502482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,128,0.08614186445871988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,7168,0.6599327723185222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,64,0.046033068497975664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,16384,32,0.053420801957448326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,64,0.06289279858271281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,32,0.061441067854563394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,6144,0.593553098042806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,16384,768,0.10302506287892658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,1024,0.14384533564249674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,768,0.12812480131785076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,16384,256,0.08873706658681234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,16384,1.1260735829671225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,16384,1.9554218292236327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,51200,3.63005116780599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,65536,4.6160425821940105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,65536,5.145550028483073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,16384,1.1984906514485678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,12288,0.8488682428995767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,12288,0.8643253326416016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,12288,1.46617062886556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,10240,0.7447381337483724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,51200,5.841687520345053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,51200,3.7360928853352866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,10240,0.7897696177164714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,10240,1.1636245727539063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,8192,0.5785034815470378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,8192,0.996020253499349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,8192,0.6041599909464519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,7168,0.5526357650756836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,6144,0.43626880645751953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,7168,0.5929450352986654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,65536,7.31003926595052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,6144,0.6885610580444336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,7168,0.8043157577514648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,6144,0.4683733304341634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,5120,0.3732778549194336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,3584,0.2823797225952148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,4096,0.32095572153727214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,5120,0.5776159922281902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,4096,0.3113162676493327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,3584,0.4276266733805339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,3584,0.29268051783243815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,3072,0.23615786234537758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,3072,0.27488533655802405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,3072,0.3454581260681152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,2560,0.2772629419962565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,2560,0.21338879267374672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,2560,0.20660479863484701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,2048,0.1785706679026286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,1536,0.1831541379292806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,2048,0.24736746152242026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,1536,0.1317344029744466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,1024,0.13129279613494874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,1536,0.15587733586629232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,1024,0.11714346408843994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,768,0.10672106742858886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,768,0.0873855988184611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,512,0.0788640022277832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,512,0.0715498685836792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,512,0.08978346983591715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,256,0.05129813353220621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,256,0.056909867127736415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,256,0.0684991995493571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,5120,0.373857053120931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,128,0.0396725336710612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,64,0.035751465956370035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,128,0.05008960167566935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,4096,0.5008576075236003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,128,0.06133759816487631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,12288,32,0.0366922656695048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,64,0.050904532273610435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,32,0.050773334503173825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,2048,0.17373013496398926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,12288,1024,0.10975893338521321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,12288,768,0.09585279623667399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,16384,1.55206298828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,51200,2.9562901814778644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,16384,0.9042826970418295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,65536,3.9390485127766928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,51200,3.278570556640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,12288,0.7177759806315105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,16384,1.0593546549479167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,12288,1.1086229960123697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,65536,4.985177612304687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,51200,4.620160929361979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,12288,0.7818357467651367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,10240,0.6080949147542317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,10240,0.9286453247070312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,10240,0.6686080296834309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,8192,0.7332629521687826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,8192,0.5181557337443035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,7168,0.43158505757649734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,7168,0.6761866887410481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,7168,0.45439573923746746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,65536,5.970683797200521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,6144,0.38535998662312826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,6144,0.5952853520711263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,6144,0.4096096038818359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,5120,0.32976531982421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,5120,0.45624853769938156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,5120,0.3389418601989746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,4096,0.2525749365488688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,4096,0.2965205192565918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,3584,0.34375572204589844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,3584,0.24830293655395508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,3072,0.2860063870747884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,4096,0.3849749247233073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,3072,0.19030507405598956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,3072,0.2215893268585205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,2560,0.17181013425191244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,2048,0.19593386650085448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,2560,0.2643786589304606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,2560,0.20183253288269043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,2048,0.13966612815856932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,1536,0.1439477284749349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,2048,0.16042666435241698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,1024,0.1127616008122762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,1536,0.12193600336710612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,1536,0.15209280649820964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,768,0.082806396484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,1024,0.10099199612935383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,512,0.06889920234680176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,512,0.06054933468500773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,256,0.04108693202336629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,256,0.05065813461939493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,256,0.06469013293584189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,128,0.03360426823298136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,128,0.04634666840235392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,64,0.030667734146118165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,64,0.0465343991915385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,10240,32,0.031565866867701214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,32,0.04623573223749797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,3584,0.23006614049275717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,1024,0.0856223980585734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,10240,768,0.07862613201141358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,8192,0.48526614507039384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,512,0.06938453515370688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,128,0.05300800005594889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,65536,3.2508906046549475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,51200,2.589953104654948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,10240,768,0.0808736006418864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,16384,1.212444814046224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,65536,3.5125386555989584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,16384,0.8196320215861002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,16384,0.8516277313232422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,51200,3.9669812520345054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,12288,0.6156981150309245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,12288,0.880362637837728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,65536,5.04049072265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,12288,0.6450282414754231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,51200,2.673878479003906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,10240,0.5240138689676921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,10240,0.5044063886006673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,10240,0.8274005254109701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,8192,0.6512522379557292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,8192,0.43674453099568683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,8192,0.43767147064208983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,7168,0.38894294102986654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,7168,0.5820181528727214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,7168,0.4133450508117676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,6144,0.3574144045511881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,6144,0.33351573944091795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,6144,0.4713706652323405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,5120,0.3729973475138346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,5120,0.2813237190246582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,4096,0.2342122713724772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,5120,0.3020970662434896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,4096,0.33209492365519205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,4096,0.2377567927042643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,3584,0.29253972371419273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,3584,0.22754666010538735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,3584,0.19932053883870443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,3072,0.17769920031229655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,3072,0.22460373242696127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,2560,0.19713600476582843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,2560,0.16466879844665527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,2560,0.1591477394104004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,1536,0.12631253401438397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,2048,0.12728746732076007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,1536,0.10605546633402507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,1536,0.11088533401489258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,1024,0.08916587034861247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,1024,0.07751359939575195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,1024,0.08794986406962077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,768,0.07205653190612793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,768,0.06830506324768067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,768,0.07080106735229492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,512,0.053667199611663816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,256,0.036962131659189865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,512,0.06019413471221924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,512,0.05536746581395467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,256,0.05156480073928833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,256,0.04884159962336222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,128,0.05026026566823324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,64,0.03631786505381267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,128,0.045449598630269365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,32,0.03630826473236084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,64,0.05038293202718099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,8192,32,0.049854934215545654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,2048,0.15305813153584796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,8192,128,0.035988267262776694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,3072,0.16294186909993488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,8192,2048,0.12130666573842366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,65536,2.966134389241536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,51200,2.31070073445638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,16384,0.7278858820597331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,51200,3.5413108825683595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,12288,0.5634677251180013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,12288,0.8393461227416992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,65536,4.359540303548178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,10240,0.4718463897705078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,10240,0.6719562530517578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,16384,1.0960469563802084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,8192,0.5125674565633138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,8192,0.40106881459554033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,7168,0.34120426177978513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,7168,0.483896541595459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,6144,0.4160736083984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,6144,0.3171327908833822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,12288,0.6322144190470378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,5120,0.36040960947672523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,5120,0.2733045260111491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,65536,3.7496106465657553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,16384,0.8038933436075846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,4096,0.2981226603190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,4096,0.19956480662027995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,3584,0.23722559611002603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,3584,0.18288319905598957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,10240,0.5523967742919922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,6144,0.3183882713317871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,51200,2.7291478474934894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,3072,0.20657173792521158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,3072,0.15454506874084473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,2560,0.17521173159281414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,2560,0.13299840291341145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,2048,0.134333864847819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,2048,0.11191253662109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,7168,0.38228158950805663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,1536,0.1067914644877116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,1536,0.09424106280008951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,8192,0.4201962788899739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,1024,0.07306666374206543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,1024,0.06967253684997558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,3072,0.16069332758585614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,768,0.05926080147425333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,768,0.05948586861292521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,1536,0.08825279871622721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,512,0.044359465440114335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,4096,0.21839893658955895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,512,0.048891735076904294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,256,0.029367466767628986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,256,0.04135359923044841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,5120,0.26335360209147135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,128,0.025489066044489545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,128,0.03691626787185669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,2560,0.14475839932759602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,64,0.022371200720469157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,64,0.03719040155410767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,7168,32,0.02331519921620687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,7168,32,0.03775466680526733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,3584,0.19037013053894042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,2048,0.11732479731241863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,65536,2.636977132161458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,65536,3.7268437703450523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,65536,2.7166409810384113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,1024,0.0704586664835612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,256,0.037112534046173096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,128,0.03182613253593445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,16384,0.6470698674519857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,16384,0.8659754435221354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,512,0.04456640084584554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,51200,2.861960601806641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,51200,2.0275253295898437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,12288,0.6410261154174804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,12288,0.5268800099690755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,51200,2.184803263346354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,10240,0.4119584083557129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,8192,0.4306026776631673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,7168,768,0.055528533458709714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,8192,0.3473173459370931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,8192,0.35015039443969725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,7168,0.4198741277058919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,7168,0.29610347747802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,7168,0.29638293584187825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,6144,0.32544746398925783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,6144,0.28184426625569664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,6144,0.2429493268330892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,5120,0.2819242795308431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,5120,0.2271029313405355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,4096,0.22308905919392905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,5120,0.21714560190836588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,10240,0.42673279444376633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,4096,0.19105706214904786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,3584,0.16367252667744953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,10240,0.5693183898925781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,3584,0.1478549321492513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,3072,0.17592959403991698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,3072,0.14171093304951984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,3072,0.13053653240203858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,2560,0.1435200055440267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,2560,0.12296533584594727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,2048,0.11721920172373454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,2048,0.09983466466267904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,1536,0.0905834674835205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,1536,0.08198933601379395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,1536,0.07886719703674316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,16384,0.6175594965616862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,1024,0.06279573440551758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,1024,0.06258879899978638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,768,0.05042346715927124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,1024,0.06843093236287436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,768,0.05337493419647217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,512,0.04088533322016398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,768,0.054926931858062744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,512,0.04433173338572184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,256,0.025838933388392132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,512,0.05014293193817139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,256,0.03616213401158651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,128,0.022268799940745036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,128,0.033504001299540204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,64,0.01953493356704712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,64,0.03350079854329427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,32,0.020087466637293497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,6144,3584,0.19695253372192384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,6144,32,0.03362666765848796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,12288,0.4772032101949056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,2560,0.11588160196940105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,4096,0.16861333847045898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,2048,0.09721706708272299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,65536,2.2279337565104167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,51200,2.3839520772298175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,128,0.03577173153559367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,6144,256,0.03840959866841634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,65536,3.0573407491048177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,16384,0.7175210952758789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,16384,0.5833920160929362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,12288,0.5222720146179199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,51200,1.7222624460856122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,12288,0.434768009185791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,12288,0.4432373364766439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,10240,0.4366517384847005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,10240,0.38390719095865883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,8192,0.3453802744547526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,10240,0.33794987996419273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,51200,1.6800618489583332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,8192,0.3255274772644043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,8192,0.28359146118164064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,7168,0.31573546727498375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,6144,0.2732096036275228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,7168,0.2697792053222656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,6144,0.24411093393961586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,5120,0.22822826703389487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,5120,0.20297600428263346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,5120,0.1803445339202881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,4096,0.1945631980895996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,4096,0.16095040639241537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,4096,0.14905920028686523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,3584,0.17517439524332684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,3584,0.14072532653808595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,3584,0.13243626753489177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,3072,0.1503328005472819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,3072,0.11476799647013347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,2560,0.12621440092722574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,2560,0.10621013641357421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,65536,2.249886830647786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,2048,0.09294826984405517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,2048,0.08354026476542155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,7168,0.2666144053141276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,2048,0.10926079750061035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,1536,0.0712767998377482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,1536,0.0728821357091268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,1536,0.078438401222229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,1024,0.05651413202285767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,1024,0.061476266384124754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,768,0.04191679954528808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,768,0.05089066823323568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,512,0.03340693314870198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,768,0.04666026830673218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,512,0.039033599694569904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,512,0.042827733357747394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,256,0.02217493255933126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,256,0.03315306703249614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,256,0.03255893389383952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,128,0.01803626616795858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,16384,0.5197482744852702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,128,0.029636265834172566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,128,0.030741333961486816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,64,0.015379200379053751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,3072,0.11986986796061198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,64,0.030294400453567506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,5120,32,0.031012266874313354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,1024,0.051178665955861416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,6144,0.20558613141377768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,5120,32,0.015495466192563376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,5120,2560,0.09525439739227295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,65536,1.9644234975179036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,51200,1.5167093912760417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,65536,2.5375902811686197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,51200,1.9907829284667968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,16384,0.5005728085835774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,12288,0.4346730550130208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,16384,0.5388821283976237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,12288,0.4042826652526855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,10240,0.36203200022379556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,10240,0.3439530690511068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,8192,0.2886624018351237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,8192,0.26457279523213706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,16384,0.5851242701212566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,7168,0.26202133496602376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,7168,0.2322528044382731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,7168,0.24097065925598143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,6144,0.23564160664876305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,6144,0.20145492553710936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,5120,0.19810880025227864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,5120,0.16883306503295897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,4096,0.14912427266438802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,5120,0.1901909351348877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,4096,0.14244693120320637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,4096,0.1485322634379069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,3584,0.13220266501108807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,3584,0.1282154639561971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,3072,0.11524373690287273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,3072,0.1131872018178304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,65536,2.2329376220703123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,2560,0.11154773235321044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,2560,0.10310080051422119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,2560,0.09956373373667399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,2048,0.08046080271402994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,51200,1.8149002075195313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,2048,0.0837557315826416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,12288,0.39348265329996746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,1536,0.06195626656214396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,10240,0.34600747426350914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,1024,0.04469226598739624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,1024,0.052680532137552895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,8192,0.2685824076334635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,768,0.03642666737238566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,768,0.043979732195536296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,6144,0.20218240420023598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,512,0.02821439901987712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,512,0.03743253151575725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,1536,0.06569919983545938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,3584,0.12290240128835042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,256,0.019997866948445638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,256,0.031856000423431396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,256,0.024964267015457155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,128,0.016155733664830526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,128,0.028398933013280232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,3072,0.10323200225830079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,128,0.02053440014521281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,64,0.013863466183344521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,4096,32,0.014230400323867798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,64,0.028189865748087566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,4096,32,0.02815786600112915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,2048,0.07600533167521159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,1536,0.05804160038630167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,1024,0.04717439810434977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,768,0.03715306520462036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,65536,1.821323776245117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,65536,2.3028841654459637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,4096,512,0.030015999078750612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,16384,0.5128341356913249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,51200,1.7384170532226562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,16384,0.4884938557942708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,51200,1.4263605753580728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,12288,0.38763411839803064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,51200,1.5024810791015626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,12288,0.3386432011922201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,10240,0.3199082692464193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,12288,0.38803094228108725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,10240,0.305677858988444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,8192,0.2804746627807617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,8192,0.24957547187805176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,7168,0.2334976037343343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,8192,0.25309227307637533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,7168,0.21893332799275717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,6144,0.19279252688090007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,6144,0.19819733301798503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,5120,0.16059306462605794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,5120,0.16109439531962078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,5120,0.1544319947560628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,4096,0.1328810691833496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,3584,0.11751893361409504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,4096,0.13124693234761556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,16384,0.4537941296895345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,3584,0.10796373685201008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,3584,0.12204159895579021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,3072,0.10165440241495768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,3072,0.10197866757710775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,2560,0.0861728032430013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,2560,0.08930133183797201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,2560,0.0803658644358317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,2048,0.06980693340301514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,2048,0.07661226590474447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,1536,0.054731734593709315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,1536,0.061794133981068936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,1536,0.05179413159688314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,1024,0.04012693166732788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,1024,0.04845759868621826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,65536,1.9273087819417316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,768,0.03921813170115153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,768,0.04590186675389608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,10240,0.3028575897216797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,7168,0.20119253794352213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,512,0.02539093295733134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,512,0.03519999980926514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,256,0.017896533012390137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,256,0.029388799269994097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,256,0.022285866737365722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,6144,0.1781503995259603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,128,0.013153066237767538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,128,0.01883946657180786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,128,0.02732586661974589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,4096,0.11926933129628499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,64,0.012157866358757019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3584,32,0.012772267063458761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,64,0.0265173335870107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3584,32,0.026276266574859618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,3072,0.08973333040873209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,2048,0.06679360071818033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,1024,0.041278934478759764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,65536,1.6655253092447917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,65536,1.8841183980305989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,51200,1.3699989318847656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,16384,0.4280373255411784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,51200,1.2935595194498697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,16384,0.45895573298136394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,51200,1.1706923166910808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,768,0.0333568016688029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,12288,0.3376650810241699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,12288,0.3266421318054199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,8192,0.21530133883158364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,10240,0.30613654454549155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3584,512,0.026910932858784993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,10240,0.2739445368448893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,7168,0.1871989409128825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,8192,0.22324906984965004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,8192,0.2135466734568278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,7168,0.19708906809488932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,7168,0.16209279696146647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,6144,0.16310079892476398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,6144,0.14209493001302081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,6144,0.18045546213785807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,5120,0.1358847935994466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,5120,0.1444757302602132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,4096,0.11028906504313152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,4096,0.11778026421864826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,3584,0.09779520034790039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,3584,0.10373226801554362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,65536,1.420913060506185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,3072,0.09675947030385336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,3072,0.10263466835021973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,16384,0.3517397244771322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,12288,0.2671157201131185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,2560,0.07102186679840088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,2560,0.066921599706014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,2560,0.08043946425120035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,2048,0.05809813340504964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,2048,0.06773973306020101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,10240,0.22625706990559896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,1536,0.04522240161895752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,1536,0.054620798428853354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,1024,0.03247146606445313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,1024,0.041611735026041666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,1024,0.03719360033671061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,768,0.026049067576726277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,768,0.035657600561777754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,768,0.03262400031089783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,512,0.020153599977493285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,5120,0.1210431973139445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,512,0.03030400077501933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,256,0.013929599523544311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,4096,0.09800000190734863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,128,0.01053013304869334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,256,0.02553066611289978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,128,0.02490239938100179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,64,0.010872532924016316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,64,0.024631466468175253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,3072,32,0.012155733505884806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,3584,0.08821439743041992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,3072,32,0.024999467531840007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,3072,0.07656426429748535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,65536,1.4923199971516927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,65536,1.4828501383463542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,1536,0.04795733292897542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,2048,0.05750399827957153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,65536,1.2639328002929688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,16384,0.33530559539794924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,16384,0.4068149248758952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,51200,1.1242655436197917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,512,0.02637760043144226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,51200,1.164532216389974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,16384,0.3197546641031901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,12288,0.25655040740966795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,256,0.021924267212549843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,51200,0.9793173472086588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,3072,128,0.0199562668800354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,10240,0.25581013361612953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,8192,0.17199467023213705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,7168,0.15143893559773763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,8192,0.22181332906087242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,7168,0.175490140914917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,6144,0.14574933052062988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,6144,0.15184106826782226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,6144,0.13672746022542318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,5120,0.1096501350402832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,5120,0.13272533416748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,4096,0.0879957358042399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,4096,0.10692373116811116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,12288,0.3054997444152832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,3584,0.07852053642272949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,3584,0.09451626936594645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,3584,0.07585600217183432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,3072,0.06799680391947428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,3072,0.08300693035125732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,3072,0.066157865524292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,2560,0.05764906803766886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,2560,0.07136639753977457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,10240,0.2123040040334066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,2048,0.0468778649965922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,12288,0.23489599227905272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,2048,0.06057386795679728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,1536,0.03643413384755452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,2048,0.049865599473317465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,10240,0.19412693977355958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,1536,0.048392534255981445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,1024,0.026180267333984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,1024,0.03698026736577352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,8192,0.159825070699056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,1024,0.03211413423220317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,768,0.02140373388926188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,512,0.016638933618863424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,768,0.03149973352750142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,512,0.022706133127212525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,512,0.027292799949645997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,256,0.011297067006429035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,7168,0.1423957347869873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,256,0.023143466313680014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,128,0.008781866232554118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,128,0.02140053311983744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,128,0.01728746692339579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,64,0.00768746683994929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,64,0.02159573237101237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,5120,0.1024949312210083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2560,32,0.02119999925295512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,4096,0.08389333089192709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,1536,0.04113279978434245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,768,0.028091732660929365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,65536,1.3172256469726562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2560,32,0.008221866687138875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,2560,0.05732160011927286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,65536,1.4352181752522788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,51200,1.0192277272542318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,16384,0.3219861348470052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,16384,0.36741545995076497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,12288,0.2463893254597982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,12288,0.28076372146606443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,51200,1.1454933166503907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,10240,0.19928213755289714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,10240,0.23737279574076334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2560,256,0.018787199258804323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,8192,0.1606943925221761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,8192,0.1930250644683838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,7168,0.14026026725769042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,7168,0.17061866124471028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,6144,0.11912213166554768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,6144,0.14666239420572919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,12288,0.24936213493347167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,5120,0.10217386881510418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,5120,0.12661973635355633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,65536,1.3255424499511719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,4096,0.08535786469777426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,16384,0.3147114753723145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,4096,0.11247786680857341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,3584,0.07763946851094564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,3584,0.09618879954020182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,6144,0.12880746523539227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,3072,0.06341653267542521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,10240,0.20870399475097656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,3072,0.08056000073750814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,2560,0.0530730684598287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,2560,0.06865599950154623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,7168,0.14403093655904134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,2048,0.04343039989471435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,2048,0.05628266731897989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,51200,1.0792672475179037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,1536,0.03589973449707031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,1536,0.04781866470972697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,8192,0.16061654090881347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,1024,0.0240831991036733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,1024,0.034983468055725095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,3072,0.06243306795756022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,768,0.019211733341217042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,768,0.03011626601219177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,4096,0.08605973720550537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,512,0.014644267161687216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,512,0.026735999186833698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,1536,0.03617173433303833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,256,0.011249066392580668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,256,0.023105067014694215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,2560,0.0562719980875651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,128,0.008389332890510559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,128,0.020662399133046468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,5120,0.10448213418324788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,64,0.007206400235493978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,64,0.02105493346850077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,2048,32,0.007880533238252004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,2048,32,0.020623999834060668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,3584,0.0748095989227295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,1024,0.02790186603864034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,65536,0.88111572265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,2048,0.045607467492421463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,65536,1.2188949584960938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,51200,0.7212736129760742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,65536,1.028440539042155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,51200,0.9498240152994791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,16384,0.24452266693115235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,16384,0.3404512087504069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,12288,0.2496789296468099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,12288,0.17928959528605143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,10240,0.1342133363087972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,10240,0.20396906534830728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,10240,0.17133439381917318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,8192,0.10860586961110433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,8192,0.16720746358235677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,256,0.014940800269444785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,128,0.012797866264979044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,7168,0.09442133108774821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,7168,0.14753173192342123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,6144,0.08160959879557292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,6144,0.12857813040415447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,512,0.018824533621470133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,16384,0.25070293744405114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,51200,0.8162666956583658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,5120,0.06782506306966146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,4096,0.055156266689300536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,5120,0.1101802666982015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,4096,0.08964160283406576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,4096,0.06877546310424805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,2048,768,0.022387200593948366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,3584,0.048844798405965166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,3072,0.042420268058776855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,3584,0.07923946380615235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,12288,0.20384532610575357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,3072,0.06839573383331299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,2560,0.036651734511057535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,2560,0.05800853172938029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,8192,0.13372373580932617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,2048,0.029930667082468672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,2048,0.04757866859436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,7168,0.12294399738311768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,1536,0.023893332481384276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,1536,0.03812053203582764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,6144,0.10689813296000164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,1024,0.017681066195170084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,5120,0.08582506974538168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,1024,0.03041279911994934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,768,0.014286933342615762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,768,0.026522666215896606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,3584,0.059589334328969325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,512,0.011384532848993937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,3072,0.051967998345692955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,512,0.023785599072774253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,512,0.01588373382886251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,256,0.00843519965807597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,256,0.020651733875274657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,128,0.006995200117429097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,2560,0.04611626863479614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,128,0.019862399498621622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,64,0.005829333265622457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,64,0.019222400585810342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1536,32,0.006156800190607706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1536,32,0.019016534090042114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,65536,0.6211893081665039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,1024,0.023124267657597862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,65536,1.1062379201253256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,51200,0.4884938557942708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,2048,0.03810773293177287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,51200,0.8689141591389975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,16384,0.2908373196919759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,768,0.01904746691385905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,12288,0.11825599670410156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,256,0.012621866663297019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,12288,0.22085226376851402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,128,0.010981333255767823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,10240,0.09945600032806397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,10240,0.1871594587961833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,8192,0.08068479696909586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,16384,0.15664960543314616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1536,1536,0.030401066939036055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,8192,0.15376426378885905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,7168,0.07038506666819254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,7168,0.1366922696431478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,6144,0.06168853441874186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,6144,0.11947306791941326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,65536,0.869975471496582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,12288,0.16565653483072917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,5120,0.054148264726003016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,5120,0.10096533298492431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,4096,0.04227946599324544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,4096,0.08126933574676513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,16384,0.21789120038350424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,3584,0.03789653380711873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,6144,0.08634239832560221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,51200,0.6733184178670247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,3072,0.03599146604537964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,3072,0.06372373501459758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,10240,0.14153599739074707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,2560,0.028835199276606244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,2560,0.05135680039723715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,7168,0.10161706606547039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,2048,0.023471999168395995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,2048,0.0415338675181071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,3584,0.07158719698588054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,1536,0.019107200702031455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,1536,0.033675734202067056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,8192,0.11585066318511963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,1024,0.014054399728775025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,1024,0.02733760078748067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,3072,0.04493546485900879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,768,0.011758933464686077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,768,0.02534293333689372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,4096,0.0589845339457194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,512,0.009342933694521587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,512,0.020716800292332967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,1536,0.02534186641375224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,256,0.0061941335598627726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,256,0.019299199183781944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,5120,0.07181013425191243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,128,0.00552106648683548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,128,0.017707733313242595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,2560,0.037886933485666914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,64,0.00510506679614385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,64,0.017577600479125977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,1024,32,0.005276800195376078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,1024,32,0.01768853267033895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,3584,0.051309867699941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,65536,0.42757333119710283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,1024,0.018900267283121743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,65536,1.0092341105143228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,2048,0.030930133660634358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,51200,0.3377653439839681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,51200,0.7914165496826172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,16384,0.1098357359568278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,16384,0.2618655999501546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,256,0.010381866494814556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,128,0.00909546713034312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,12288,0.08138453165690104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,512,0.013212800025939941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,12288,0.20071999231974283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,10240,0.06878933111826578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,10240,0.16945172945658366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,8192,0.05631359815597534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,8192,0.13897706667582194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,7168,0.04970666567484538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,1024,768,0.016169599692026772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,7168,0.12478720347086589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,6144,0.04317973454793294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,6144,0.10882240136464436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,65536,0.8159530639648438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,5120,0.03744639952977498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,5120,0.09636800289154053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,16384,0.2157130718231201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,4096,0.02986133297284444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,51200,0.6567509333292644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,4096,0.07445333003997803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,3584,0.02643199960390727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,3584,0.06413546800613404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,12288,0.1613653341929118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,3072,0.02225173314412435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,3072,0.05469013452529907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,10240,0.13590933481852213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,2560,0.019511467218399046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,2560,0.04503573179244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,6144,0.08460693359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,2048,0.01646293302377065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,2048,0.037222401301066084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,7168,0.09923733075459798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,1536,0.013288533687591553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,1536,0.031148799260457355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,8192,0.11300053596496581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,1024,0.010151466727256775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,1024,0.024676267306009927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,5120,0.06954452991485596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,768,0.008693333466847737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,768,0.022600533564885457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,3072,0.0446730653444926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,512,0.0071168000499407455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,512,0.020071466763814293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,4096,0.058133331934610996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,256,0.0057888001203536986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,256,0.01882986625035604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,3584,0.05093013445536295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,128,0.004954666892687479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,128,0.017427200078964235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,1536,0.024099200963974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,64,0.0044725333650906885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,64,0.0167797327041626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,768,32,0.004757333298524221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,768,32,0.01704746683438619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,2560,0.03657279809315999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,65536,0.31510613759358724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,2048,0.030461867650349934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,65536,0.9934911727905273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,1024,0.017463467518488564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,51200,0.25108799934387205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,16384,0.10560533205668132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,16384,0.2617215951283773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,51200,0.7820266723632813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,12288,0.07760852972666422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,12288,0.19962879816691081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,256,0.009286399682362874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,10240,0.06842666467030843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,512,0.011730133493741354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,128,0.007866666714350382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,8192,0.054345599810282384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,10240,0.16949013074239094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,8192,0.13961812655131023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,7168,0.04835626681645711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,768,768,0.01453439990679423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,7168,0.12496000130971271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,6144,0.041843199729919435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,6144,0.10827199618021648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,12288,0.11949546337127685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,5120,0.03471146821975708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,65536,0.5793504079182943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,5120,0.09113600254058837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,4096,0.029239465792973835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,4096,0.07298133373260499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,16384,0.15786560376485187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,3584,0.026855466763178508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,3584,0.06239786545435587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,51200,0.46446720759073895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,3072,0.023252266645431518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,3072,0.05313920180002848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,6144,0.06205546855926514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,2560,0.019826134045918785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,2560,0.04288746515909831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,10240,0.10112746556599934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,2048,0.01639786660671234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,2048,0.035345065593719485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,7168,0.07162773609161377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,8192,0.08169706662495932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,1536,0.012939733266830445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,1536,0.02966933250427246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,1024,0.009687466422716777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,1024,0.023826134204864503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,4096,0.04346453348795573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,768,0.00788373351097107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,3072,0.03346773386001587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,768,0.021859200795491536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,512,0.006146133442719777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,512,0.019271467129389444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,5120,0.05295466581980387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,256,0.004735999802748362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,256,0.017753599087397258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,3584,0.03882026672363281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,128,0.004221866528193155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,128,0.01698346734046936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,2560,0.028409600257873535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,64,0.003973333289225897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,64,0.016335999965667723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,512,32,0.004212266703446706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,512,32,0.016457600394884746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,1536,0.01920213301976522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,65536,0.18701012929280597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,65536,0.8938784281412759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,51200,0.15004693667093913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,2048,0.023461333910624185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,51200,0.7020287831624349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,16384,0.05714773337046305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,16384,0.23648106257120766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,256,0.008067200084527333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,1024,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,12288,0.04611733357111613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,512,0.010345600048700969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,10240,0.03588053385416667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,12288,0.18259199460347492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,10240,0.15433386166890461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,128,0.007066666583220164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,8192,0.028830933570861816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,8192,0.12581546306610109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,7168,0.026616533597310383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,512,768,0.012066133817036946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,7168,0.11138986746470134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,6144,0.023397332429885863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,6144,0.09696533679962158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,65536,0.5757151921590169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,5120,0.016107733050982155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,5120,0.081167999903361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,51200,0.4486367861429851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,4096,0.013078400492668152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,16384,0.1504458745320638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,4096,0.06288640101750692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,3584,0.011927466591199238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,3584,0.05270933310190836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,12288,0.1129856030146281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,3072,0.01076800028483073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,3072,0.04416853189468384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,10240,0.09527146816253662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,2560,0.009674666325251262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,2560,0.03707306782404582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,7168,0.06831680138905843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,8192,0.07758933703104655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,2048,0.008186666667461396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,2048,0.031137067079544067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,1536,0.007121066749095917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,1536,0.026343466838200887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,6144,0.059171199798583984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,1024,0.0058335999647776285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,1024,0.022387200593948366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,5120,0.050385065873463954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,768,0.005273599922657013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,768,0.020501333475112914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,3072,0.031420799096425374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,512,0.004671999812126159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,512,0.018658133347829182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,4096,0.04158080021540324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,256,0.003922133396069208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,256,0.017129600048065186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,3584,0.036858665943145755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,128,0.003656533360481262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,128,0.01636373301347097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,1536,0.017491199572881064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,64,0.003389866650104523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,64,0.015640532970428465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,256,32,0.0035360001027584078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,256,32,0.015619200468063355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,2560,0.02595413327217102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,65536,0.17018879254659017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,2048,0.021622399489084877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,65536,0.8848128000895181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,51200,0.13670825958251953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,1024,0.013266133268674216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,16384,0.048794666926066085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,51200,0.371560541788737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,16384,0.2301952044169108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,12288,0.03929813305536906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,16384,0.12597119808197021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,51200,0.696455446879069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,12288,0.1767189343770345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,10240,0.0291594664255778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,10240,0.14976320266723633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,256,0.007236266632874806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,768,0.01097706655661265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,8192,0.02396799921989441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,128,0.006163200239340464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,65536,0.47006826400756835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,7168,0.021437867482503255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,8192,0.06523733139038086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,6144,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,256,512,0.00925439993540446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,7168,0.10757973194122314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,5120,0.011346133550008138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,6144,0.09381226698557535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,5120,0.07790826956431071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,5120,0.042430933316548666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,4096,0.009521067142486572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,4096,0.060429867108662924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,4096,0.034434131781260174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,3584,0.009071999788284301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,3584,0.050964268048604336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,3072,0.008036266764005024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,3072,0.0420085350672404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,3072,0.025821866591771443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,2560,0.008217599987983704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,2560,0.0349397341410319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,2560,0.022286933660507203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,2048,0.007288533449172974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,12288,0.09657066663106283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,2048,0.030601600805918377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,2048,0.01875093380610148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,1536,0.006186666587988535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,1536,0.01578986644744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,1536,0.02571733395258586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,1024,0.005126399795214335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,1024,0.021870932976404824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,8192,0.12187093098958332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,768,0.004817066589991251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,10240,0.08022186756134034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,768,0.02031893332799276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,512,0.0041514667371908825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,768,0.0102293332417806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,256,0.0036415999134381616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,512,0.018313600619633993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,256,0.016754132509231568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,128,0.003230933348337809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,256,0.007302399973074596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,6144,0.049615999062856034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,64,0.003054933249950409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,128,0.00631466656923294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,128,0.016078933080037435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,128,32,0.00310506671667099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,64,0.015256533026695251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,128,32,0.01508799990018209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,65536,0.16058133443196615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,51200,0.1274453322092692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,16384,0.044499198595682785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,3584,0.03138239979743958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,12288,0.03509226640065511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,10240,0.03041599988937378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,16384,0.23067092895507812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,12288,0.1761770725250244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,8192,0.025705599784851076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,7168,0.02322346568107605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,10240,0.1486848036448161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,51200,0.6855295817057292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,6144,0.015289599696795145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,8192,0.12088533242543538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,5120,0.01027413308620453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,7168,0.10715093612670898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,4096,0.008595200379689534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,65536,0.8790047963460287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,3584,0.00800960014263789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,6144,0.0928757349650065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,4096,0.05931626558303833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,3072,0.007430399954319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,3584,0.04955413341522217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,2560,0.007539199789365132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,2048,0.006805333495140076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,2560,0.03455253442128499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,1536,0.005835733314355215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,2048,0.02988906701405843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,1024,0.004643199841181437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,1536,0.02540266712506612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,768,0.004223999877770742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,1024,0.022245333592096964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,512,0.003755733370780945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,768,0.0195743997891744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,256,0.0034314667185147605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,128,0.003190399954716364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,256,0.016570666432380678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,64,0.0030261332790056865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,128,0.015658666690190635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,64,32,0.0030400000512599947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,64,0.015147733688354491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,32,0.015403733650843302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,7168,0.05728960037231445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,65536,0.15689813296000163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,51200,0.12398826281229655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,16384,0.04404266675313313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,16384,0.2284693400065104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,51200,0.6845279693603515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,65536,0.8761802673339844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,5120,0.0772874673207601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,12288,0.03478613297144572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,10240,0.030114134152730305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,3072,0.04189866781234741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,8192,0.02459413409233093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,7168,0.021946666638056438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,12288,0.17628693580627441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,10240,0.14832746187845866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,8192,0.1212725321451823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,512,0.009032533566157023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,64,512,0.018092799186706542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,5120,0.00983786682287852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,7168,0.10650239785512287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,6144,0.0923466682434082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,5120,0.07579519748687744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,3584,0.007739733159542084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3328,128,1024,0.011869866649309795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,3072,0.007051733136177063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,4096,0.05860586563746134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,3584,0.04902079900105794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,2560,0.006323199967543285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,2048,0.005529599885145823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,3072,0.04043200016021729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,1536,0.00491839994986852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,2560,0.03457280000050862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,2048,0.02918933431307475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,768,0.003903999924659729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,1536,0.025038933753967284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,1024,0.02151893377304077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,768,0.020522665977478028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,256,0.0031839999059836066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,512,0.017870932817459106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,128,0.002898133297761281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,256,0.016217600305875143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,64,0.0027583998938401537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,128,0.015734400351842245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,32,0.0028351999819278715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,64,0.015245866775512696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3328,32,32,0.015101866920789084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,6144,0.0170741339524587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,4096,0.008400000135103862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,1024,0.004188799858093261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3328,32,512,0.0035082665582497918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,16384,4.824166361490885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,12288,3.7698229471842444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,16384,6.401526387532551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,12288,6.673571268717448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,16384,8.97027587890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,10240,2.9741055806477865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,10240,5.497834777832031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,10240,4.038712565104166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,8192,2.4692459106445312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,8192,4.883308919270833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,12288,4.992655944824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,51200,15.922451782226563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,7168,2.4266026814778643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,7168,4.566813659667969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,7168,3.0693430582682293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,6144,1.8091776529947918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,6144,3.388908894856771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,8192,3.0949429829915363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,6144,2.364368947347005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,5120,1.5347668965657553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,4096,1.2231413523356118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,5120,2.880968475341797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,5120,1.9001771291097005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,4096,2.289867655436198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,4096,1.5809611002604167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,3584,1.087829335530599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,3072,0.9448640187581381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,3584,1.981488037109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,3072,1.6797524770100911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,2560,0.8335776011149088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,2560,1.4715039571126303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,2560,1.0116064071655273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,2048,1.143936030069987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,2048,0.8069013595581055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,2048,0.8273087819417319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,1536,0.5290357271830242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,1536,0.842846934000651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,1024,0.5819498697916667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,1024,0.3949258804321289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,1024,0.5606709162394206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,1536,0.9553397496541342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,768,0.3203488032023112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,768,0.4610165278116862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,3584,1.391216023763021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,512,0.3767189343770345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,512,0.22545386950174967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,256,0.22676374117533366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,512,0.3902912139892578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,3072,1.1484362284342446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,128,0.1759786605834961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,256,0.19373973210652667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,128,0.17717866897583007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,256,0.30769707361857096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,64,0.16374613444010416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,64,0.17524587313334147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,32,0.16027520497639974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,65536,32,0.17094400723775227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,128,0.34851306279500327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,768,0.443887996673584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,65536,51200,28.06977132161458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,65536,51200,27.37556559244792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,16384,4.013637288411458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,16384,7.03231913248698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,51200,11.703472900390626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,65536,15.4271728515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,12288,5.584570821126302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,16384,5.250553385416667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,12288,3.1726539611816404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,10240,2.5307273864746094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,12288,4.1310368855794275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,10240,4.816964213053385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,8192,1.8200021107991535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,51200,22.72524210611979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,8192,3.4688064575195314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,8192,2.4718505859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,51200,22.135298665364584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,7168,1.6553760528564454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,10240,3.0962473551432295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,7168,3.1433759053548176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,7168,2.4474143981933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,6144,1.4109823862711588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,6144,1.9337151845296223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,6144,2.68959477742513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,5120,1.1918219248453776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,5120,2.1987444559733076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,65536,28.532244873046874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,4096,1.0510496139526366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,5120,1.8203275044759113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,4096,1.7760011037190755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,3584,0.8181151707967123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,4096,1.2435626983642578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,3584,1.6186368306477863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,3072,0.7284778594970703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,2560,0.6907871882120769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,3584,1.0335903803507487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,65536,29.242177327473957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,2560,1.1633706410725912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,3072,1.429401652018229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,2560,0.8456223805745443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,2048,0.5410079956054688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,2048,0.9514837265014648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,1536,0.4312138557434082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,2048,0.6842666625976562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,1024,0.48527679443359373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,1536,0.6136575698852539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,512,0.2542634646097819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,1536,0.6683093388875325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,1024,0.3174880027770996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,512,0.1883135954538981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,256,0.1560266653696696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,768,0.24489386876424155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,768,0.35390186309814453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,256,0.15440533955891927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,1024,0.4855690638224284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,768,0.42695147196451827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,128,0.14017279942830402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,128,0.14413866996765137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,64,0.12978239854176837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,512,0.27704426447550456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,256,0.23631466229756676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,64,0.14071146647135419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,51200,32,0.12801493008931478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,51200,32,0.13924585978190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,3072,0.869043223063151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,16384,2.3174112955729167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,51200,128,0.2212981383005778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,16384,1.2987093607584634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,51200,4.071430460611979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,16384,1.5762058258056642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,65536,5.178050231933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,51200,5.355855814615885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,12288,0.959053866068522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,12288,1.6625695546468098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,12288,1.1944405873616537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,10240,0.8067317326863608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,10240,0.999449602762858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,51200,6.845511372884114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,8192,1.1106751759847007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,10240,1.397994613647461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,8192,0.6658463795979818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,8192,0.7747264226277669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,7168,0.5704309463500976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,7168,0.7358421325683594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,65536,8.207388814290365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,6144,0.835197893778483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,6144,0.5165514628092448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,6144,0.5995626449584961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,5120,0.45578667322794597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,4096,0.34342934290568033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,5120,0.6613610585530598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,4096,0.5621962865193685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,5120,0.5203754742940266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,3584,0.3050069491068522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,3584,0.49533545176188154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,3584,0.37666667302449547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,65536,8.871996053059895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,3072,0.27694613138834634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,3072,0.42868480682373045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,3072,0.31707626978556314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,7168,1.0064586639404296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,2560,0.3642730712890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,2560,0.27499411900838217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,2048,0.2863061269124349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,2048,0.1904586633046468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,2560,0.23412159283955894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,1536,0.2312490622202555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,2048,0.21960426966349283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,1536,0.19555733998616537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,1024,0.11388266881306966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,1024,0.14636054039001464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,1536,0.1513674736022949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,768,0.11656746864318848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,1024,0.15137599309285482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,768,0.09830400149027506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,512,0.08645013173421225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,512,0.07511999607086181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,768,0.1255189339319865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,512,0.1005130688349406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,256,0.05902933279673258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,256,0.06327360073725383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,128,0.04870293140411377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,256,0.08582186698913574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,64,0.04493333498636882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,128,0.05850453376770019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,128,0.07749546368916829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,16384,32,0.04492799838383992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,64,0.05863680044809977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,16384,32,0.05764586528142294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,16384,4096,0.39251200358072913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,16384,1.7154986063639324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,51200,3.860291290283203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,51200,3.9298772176106773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,65536,4.752809651692709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,16384,1.2273386637369792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,16384,0.9850005467732748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,12288,0.7771669387817383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,12288,1.3216138203938803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,12288,0.933837890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,51200,5.812959798177084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,10240,0.76866455078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,10240,1.0904127756754556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,8192,0.8580885569254557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,8192,0.5592810948689778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,8192,0.5982378641764323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,7168,0.46753921508789065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,7168,0.7333824157714843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,7168,0.5575541178385417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,6144,0.4220416069030762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,6144,0.6275242487589519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,6144,0.45322879155476886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,65536,6.925191243489583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,5120,0.3456426620483398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,10240,0.6132511774698893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,5120,0.5472672144571941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,4096,0.27254292170206706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,4096,0.4424917221069336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,3584,0.22692373593648277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,4096,0.318945058186849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,3584,0.354089609781901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,3584,0.3177386601765951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,3072,0.3002570788065592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,3072,0.2287519931793213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,3072,0.23220693270365395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,2560,0.19797760645548504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,2560,0.2535456021626791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,2048,0.2125333309173584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,2048,0.1580949306488037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,2560,0.2208789348602295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,2048,0.16982827186584473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,1536,0.16651520729064942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,1536,0.1226698637008667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,1024,0.11955839792887371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,1024,0.09492373466491699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,1024,0.1077386697133382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,768,0.09941653410593668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,1536,0.15162560145060222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,768,0.08019626935323079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,512,0.06662613153457642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,256,0.04487466812133789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,768,0.1033567984898885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,512,0.05999039808909098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,256,0.05182506640752157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,512,0.09254293441772461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,128,0.03746986786524455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,256,0.06931946277618409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,64,0.03304106593132019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,128,0.060711467266082765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,128,0.0473909338315328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,12288,32,0.03257173299789429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,64,0.046435201168060304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,65536,5.200154622395834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,12288,32,0.04676906665166219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,16384,1.4423221588134765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,16384,0.9192160288492838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,12288,5120,0.38212267557779944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,51200,2.8140724182128904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,51200,3.205626678466797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,65536,3.586837259928385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,16384,0.9688554763793945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,51200,4.36588389078776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,12288,0.6582069396972656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,65536,4.392336018880209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,12288,0.7505034764607748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,12288,1.0051690419514974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,8192,0.465559450785319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,8192,0.5012928009033203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,10240,0.6253376007080078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,10240,0.5486101150512696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,8192,0.6650453567504883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,10240,0.8294357299804688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,65536,5.616034952799479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,7168,0.4069301287333171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,7168,0.5771637598673502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,7168,0.45209493637084963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,6144,0.35307200749715173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,6144,0.5104821205139161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,5120,0.2917930603027344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,6144,0.3888117472330729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,4096,0.24060586293538413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,5120,0.41028906504313156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,4096,0.3470421473185221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,5120,0.35732800165812173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,4096,0.3093834559122721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,3584,0.21267414093017578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,3584,0.29575039545694987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,3072,0.2599712053934733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,3584,0.23813652992248535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,3072,0.2000607967376709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,3072,0.2229695955912272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,2560,0.25109546979268393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,2048,0.19352960586547852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,2560,0.1632746696472168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,2048,0.13912960688273113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,1536,0.10984426339467365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,2048,0.15126506487528485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,2560,0.1741088072458903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,1536,0.15915946960449218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,1024,0.10768427054087322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,1024,0.08346880276997884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,768,0.06904853185017903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,1024,0.10327253341674805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,768,0.08152320384979247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,512,0.06151253382364909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,256,0.03783786694208781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,512,0.06933120091756186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,256,0.046938665707906085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,256,0.056194134553273524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,128,0.04407573143641154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,64,0.02984106739362081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,32,0.02940160036087036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,64,0.04227413336435954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,10240,32,0.04448533455530802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,1536,0.14086079597473145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,768,0.07535999615987142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,512,0.06667733192443848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,10240,128,0.032671999931335446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,10240,128,0.05354453325271606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,16384,0.7261418660481771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,16384,1.109332275390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,16384,0.8092746734619141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,12288,0.8787477493286133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,65536,3.018759409586589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,51200,2.3158645629882812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,12288,0.5709397633870442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,51200,2.610759480794271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,12288,0.6208213170369465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,10240,0.48809172312418625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,65536,4.5666656494140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,51200,3.5441078186035155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,8192,0.38401387532552084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,8192,0.5202720006306966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,7168,0.4728821436564128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,7168,0.35753173828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,7168,0.3650954564412435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,6144,0.4043882687886556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,6144,0.30908266703287757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,6144,0.31903146107991537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,5120,0.3338442802429199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,5120,0.2737237294514974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,4096,0.26927572886149087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,4096,0.21558186213175454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,3584,0.23915413220723472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,3584,0.18711039225260417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,3584,0.20385173161824546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,3072,0.20484372774759926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,3072,0.1600714683532715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,2560,0.17216639518737792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,65536,3.4829760233561196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,10240,0.6769856135050456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,5120,0.25721492767333987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,2560,0.13496747016906738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,2560,0.16667946179707843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,2048,0.11161173184712728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,2048,0.12390293280283611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,1536,0.11633386611938476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,1536,0.09063039620717367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,1024,0.07682560284932455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,10240,0.504966386159261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,1024,0.06990613142649332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,768,0.06092906792958578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,1024,0.08962773482004802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,768,0.059639465808868405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,512,0.04614293177922567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,512,0.048716799418131514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,512,0.05661760171254476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,8192,0.40373760859171554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,256,0.03136853377024333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,256,0.04680746793746948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,128,0.025172267357508344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,2048,0.1420479933420817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,128,0.0380298654238383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,64,0.023647999763488768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,64,0.03805653254191081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,8192,32,0.023912533124287923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,32,0.037970133622487384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,4096,0.21618773142496744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,3072,0.16581013997395833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,8192,256,0.039060266812642415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,1536,0.09819200038909912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,768,0.06734399795532227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,8192,128,0.04322346846262614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,16384,0.9684384028116861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,65536,2.6998250325520834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,16384,0.6771946589152018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,51200,2.0824863433837892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,16384,0.717852783203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,12288,0.7287808100382487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,51200,3.1740992228190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,65536,4.146700795491537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,12288,0.5226346651713054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,12288,0.5396362940470378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,10240,0.4639466603597005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,10240,0.6430496215820313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,10240,0.4378634770711263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,8192,0.3659167925516764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,7168,0.31133546829223635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,8192,0.3517951965332031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,7168,0.412169615427653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,8192,0.4605418523152669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,7168,0.31767145792643225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,6144,0.26351466178894045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,5120,0.29719041188557943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,6144,0.35924161275227867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,5120,0.2316970666249593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,4096,0.2334218660990397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,4096,0.1995946725209554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,3584,0.20560639699300132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,3584,0.17897920608520507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,3584,0.15910293261210123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,3072,0.17773973147074382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,3072,0.14316587448120116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,5120,0.22505386670430502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,2560,0.15032213528951008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,3072,0.14813653628031415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,65536,2.942609151204427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,2048,0.12355519930521648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,1536,0.09554666678110758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,2560,0.12597866853078205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,2048,0.10218026638031005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,2048,0.11923413276672364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,1536,0.08525439898173014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,1536,0.08699413140614828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,51200,2.2151262919108072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,1024,0.06742933591206869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,768,0.05472106536229452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,1024,0.0702677329381307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,768,0.06786453723907471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,512,0.0416159987449646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,512,0.0448085347811381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,256,0.02728533347447713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,512,0.06530453364054362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,256,0.03794133265813192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,256,0.04211733341217041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,128,0.023853866259257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,64,0.021410133441289267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,128,0.03910826841990153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,128,0.03476800123850505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,32,0.021492266654968263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,64,0.036848000685373944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,32,0.036101333300272626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,7168,2560,0.12296319802602132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,6144,0.27456318537394203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,7168,1024,0.06787839730580648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,768,0.06478186845779418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,7168,4096,0.1880224068959554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,16384,0.7646720250447591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,16384,0.5843477249145508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,65536,2.321763102213542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,12288,0.5742581049601237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,12288,0.45412267049153643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,51200,2.70273920694987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,65536,3.4347999572753904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,10240,0.5011509259541829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,10240,0.3763584136962891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,8192,0.4168394724527995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,51200,1.8546783447265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,8192,0.30365120569864906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,7168,0.3448800086975098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,7168,0.2912853240966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,6144,0.2891445477803548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,6144,0.24476906458536782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,12288,0.5913663864135742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,5120,0.24646933873494467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,16384,0.8125514984130859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,5120,0.20364267031351724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,4096,0.1984277407328288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,4096,0.16548479398091634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,10240,0.4991360028584798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,3584,0.1746026674906413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,65536,3.925939178466797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,3584,0.14749973615010578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,3072,0.15161387125651044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,51200,2.589527384440104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,3072,0.12956373691558837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,2560,0.12771413326263428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,2560,0.11085866292317707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,7168,0.3606015841166178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,2048,0.10455466906229656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,2048,0.09172159830729167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,8192,0.391427199045817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,1536,0.08162240187327066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,1536,0.07750613689422607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,6144,0.30064214070638023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,1024,0.057334399223327635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,1024,0.057955201466878256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,3072,0.14495679537455242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,4096,0.20440106391906737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,768,0.04589866797129313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,768,0.05110506614049276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,512,0.03540800015131633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,512,0.04195733467737834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,5120,0.2506986618041992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,1536,0.07622826894124349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,256,0.024094933271408082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,256,0.034517331918080645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,128,0.020170666774113975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,128,0.03232959906260173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,2560,0.12802666823069256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,64,0.017593600352605186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,64,0.03216213385264079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,6144,32,0.018145066499710084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,6144,32,0.03211306730906169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,3584,0.1731594721476237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,1024,0.059774935245513916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,2048,0.1016202688217163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,65536,2.9067339579264324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,65536,2.0615360260009767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,65536,2.1181599934895834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,51200,2.2500139872233076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,16384,0.7114847819010417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,51200,1.6247423807779948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,128,0.026586665709813433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,51200,1.7019189198811848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,256,0.03209386666615804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,12288,0.4002271970113118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,16384,0.5752874374389648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,12288,0.4427317301432292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,512,0.03810880184173584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,10240,0.4039733250935872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,10240,0.3405376116434733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,8192,0.3482410748799642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,8192,0.2701141357421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,8192,0.2869642575581869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,7168,0.2913653373718262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,7168,0.26658347447713215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,7168,0.2369248072306315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,6144,0.25552426973978676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,6144,0.22976746559143066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,5120,0.18788159688313802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,6144,768,0.04749546845753987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,4096,0.17355732917785643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,5120,0.1891274611155192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,4096,0.14081172943115233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,4096,0.14900693893432618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,3584,0.15081920623779296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,12288,0.496015993754069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,3584,0.1243829329808553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,3072,0.13054613272349042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,3072,0.11061866283416748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,3072,0.11279146671295166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,2560,0.11196906566619873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,2560,0.09668160279591878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,2048,0.0928938627243042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,2048,0.08229333559672038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,2048,0.09008853435516358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,1536,0.067740797996521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,1536,0.07333866755167642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,1024,0.05289599895477295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,1024,0.05541653235753378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,5120,0.20742613474527993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,1024,0.05554560025533041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,768,0.03993599812189738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,768,0.04728000164031983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,768,0.04645546674728394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,512,0.03086613416671753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,512,0.03813226620356242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,16384,0.5561941146850586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,256,0.021065600713094077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,3584,0.13362560272216797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,256,0.03107200066248576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,128,0.01650773286819458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,128,0.027894399563471478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,1536,0.06575359900792441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,10240,0.32788801193237305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,64,0.014206932981808982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,5120,32,0.015082666277885437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,64,0.02877226670583089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,5120,32,0.02873493234316508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,6144,0.20458559989929198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,2560,0.09681920210520426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,65536,1.8169439951578776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,65536,2.3134251912434896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,512,0.039049601554870604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,256,0.03248959978421529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,5120,128,0.0295413335164388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,16384,0.5343744277954101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,51200,1.8281888326009113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,16384,0.46311254501342775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,12288,0.391320546468099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,12288,0.373417599995931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,10240,0.33333759307861327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,51200,1.4308544158935548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,12288,0.42524054845174153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,10240,0.30254507064819336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,8192,0.2776426633199056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,8192,0.25181333223978675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,7168,0.233079465230306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,7168,0.2333087921142578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,6144,0.2011338710784912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,6144,0.20359999338785806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,5120,0.1805173397064209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,5120,0.16702292760213217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,5120,0.17743786176045734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,4096,0.137662935256958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,4096,0.13234559694925946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,65536,2.382677459716797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,3584,0.13568533261617025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,16384,0.5553621292114258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,3584,0.12729067007700604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,3072,0.11513173580169678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,3072,0.10601387023925782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,10240,0.35148585637410484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,8192,0.2750730514526367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,2560,0.08938026428222656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,2560,0.08749226729075113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,2048,0.07253013451894125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,7168,0.24964373906453452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,2048,0.07569706439971924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,6144,0.20943679809570312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,51200,1.8023626963297528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,1536,0.056281598409016934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,1536,0.06184853315353393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,1024,0.04075093269348144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,1024,0.04603413343429565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,1024,0.04835306803385417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,768,0.033086933692296344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,768,0.04176533222198486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,768,0.036720001697540285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,512,0.025730133056640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,512,0.03505599896113078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,512,0.029547733068466187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,256,0.018173867464065553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,256,0.03056959907213847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,128,0.014618666966756186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,128,0.02847786744435628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,4096,0.14339626630147298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,64,0.013846400380134582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,64,0.028641066948572796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,4096,32,0.015006933609644571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,4096,32,0.028784000873565675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,3584,0.12616106669108074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,3072,0.10766613483428955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,2560,0.09329066276550294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,2048,0.07466026941935221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,65536,1.9934090932210284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,1536,0.056927998860677086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,256,0.024462932348251344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,16384,0.45305919647216797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,4096,128,0.02026026646296183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,65536,1.7022528330485027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,16384,0.4217429478963216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,12288,0.33517653147379556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,51200,1.4891690572102865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,51200,1.300866190592448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,16384,0.501637331644694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,12288,0.33860054016113283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,12288,0.35485760370890296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,10240,0.2811978658040365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,8192,0.24217066764831544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,10240,0.3113631884256999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,8192,0.229911470413208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,7168,0.19943572680155436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,6144,0.17701652844746907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,7168,0.21395626068115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,6144,0.16706666946411133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,5120,0.14741867383321125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,5120,0.14833812713623046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,5120,0.15605546633402506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,4096,0.11933440367380779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,4096,0.11888960202534993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,10240,0.30685014724731446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,3584,0.10501226584116619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,3584,0.10453973611195881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,65536,2.04410769144694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,51200,1.6503584543863934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,3072,0.10107519626617431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,3072,0.10721173286437988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,8192,0.23746879895528158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,7168,0.21880745887756348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,2560,0.07880853017171224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,6144,0.18587093353271483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,2560,0.0813215970993042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,2048,0.06557439963022868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,1536,0.05249706506729126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,2048,0.07037546634674072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,1536,0.05081280072530111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,1536,0.05856639941533407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,1024,0.03578346570332845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,1024,0.04614506562550862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,1024,0.04074346621831258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,768,0.030279467503229778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,512,0.02272746761639913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,768,0.03341226577758789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,768,0.037970133622487384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,256,0.015796266992886863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,512,0.03172159989674886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,256,0.026001065969467163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,256,0.02226240038871765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,128,0.012277332941691081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,128,0.02526400089263916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,64,0.01135040024916331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,64,0.025131734212239583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3584,32,0.011482666929562886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,4096,0.12634027004241943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3584,32,0.02526293396949768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,3584,0.11058239936828614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,65536,1.4919211069742837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,65536,1.6172276814778646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,3072,0.09579306443532308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,2560,0.0809397300084432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,2048,0.066157865524292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,16384,0.37618986765543616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,16384,0.3853973388671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,512,0.02621440092722575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,51200,1.2329867045084635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,51200,1.1956991831461588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3584,128,0.01774186690648397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,12288,0.28324693044026694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,16384,0.44817066192626953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,12288,0.29438826243082683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,8192,0.19611627260843914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,10240,0.24068479537963866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,10240,0.25149653752644857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,51200,1.394821294148763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,7168,0.17210453351338703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,7168,0.18097920417785646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,6144,0.15196693738301595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,6144,0.15664854049682617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,5120,0.12422719796498616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,5120,0.13277119795481365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,8192,0.20247146288553872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,4096,0.09923199812571207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,4096,0.10559146404266358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,65536,1.8293824513753254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,3584,0.09751146634419759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,3584,0.10281813144683838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,12288,0.31192960739135744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,3072,0.07528426647186279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,10240,0.2714442571004232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,3072,0.08505813280741373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,3072,0.08204159736633301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,2560,0.06361173391342163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,8192,0.21501866976420084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,2560,0.0732266664505005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,7168,0.19213120142618817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,2048,0.05264960130055746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,2560,0.07242986361185709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,2048,0.06238293250401815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,1536,0.04099733432133992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,2048,0.05900266567866007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,1536,0.051083731651306155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,1024,0.029275733232498168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,768,0.023902932802836098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,1024,0.03968000014623006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,768,0.03285119930903117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,512,0.01858560045560201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,6144,0.16457173029581706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,512,0.022445867458979286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,512,0.029045333464940388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,256,0.012732799847920737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,256,0.02290133237838745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,128,0.009665066997210186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,5120,0.13887359301249186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,128,0.021591466665267945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,64,0.008436266581217449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,4096,0.11351040204366047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,64,0.021356799205144248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,3072,32,0.008764800429344178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,3072,32,0.021347200870513915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,3584,0.09784746964772542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,65536,1.4120330810546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,1536,0.044905598958333334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,1024,0.03631786505381267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,768,0.02860693335533142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,256,0.01876586675643921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,65536,1.3845855712890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,3072,128,0.015561599532763162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,16384,0.31982078552246096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,16384,0.29737812678019204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,51200,1.0436831792195638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,12288,0.23856852849324545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,51200,1.0731807708740235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,12288,0.28527892430623375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,12288,0.23101332982381187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,10240,0.23396479288736977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,10240,0.22789546648661294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,8192,0.16413973172505697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,10240,0.1898965358734131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,8192,0.19628373781840008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,7168,0.1417450745900472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,7168,0.13788800239562987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,7168,0.16674772898356122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,6144,0.12499946753184002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,6144,0.14010666211446127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,5120,0.10381440321604411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,5120,0.12406400044759114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,4096,0.08411733309427896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,5120,0.10171199639638265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,4096,0.10077653725941975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,3584,0.07334400018056234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,3584,0.090447998046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,3584,0.07435519695281982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,3072,0.0648138682047526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,3072,0.07758933703104655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,2560,0.053734401861826576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,16384,0.3620181401570638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,2560,0.06722133159637451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,2560,0.05704106489817301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,2048,0.044062932332356766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,2048,0.05598933299382528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,1536,0.03418240149815877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,1536,0.04485119978586833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,6144,0.12057279745737712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,1024,0.024793599049250284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,1024,0.03466026782989502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,4096,0.08554133574167887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,768,0.020150399208068846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,768,0.03138986627260844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,51200,0.9347050348917643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,512,0.016911999384562174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,512,0.028442666927973433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,512,0.02573866645495097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,65536,1.2188159942626953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,256,0.024794665972391765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,8192,0.1565717379252116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,128,0.008266666531562805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,128,0.020576000213623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,1024,0.03152533372243245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,128,0.016657066345214844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,64,0.007500799993673961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,32,0.007852800190448761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,64,0.02039253314336141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2560,32,0.020333866278330483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,3072,0.06660693486531576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2560,256,0.012075733145078022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,2048,0.04903573195139567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,65536,1.0455392201741538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,1536,0.039724798997243245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,65536,1.2618335723876952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,16384,0.2658090591430664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,51200,0.8531466801961264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,16384,0.3280245463053385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,65536,1.3485333760579428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,12288,0.22073386510213217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,51200,0.9906496047973633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,12288,0.24985706011454262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,768,0.027478400866190595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,10240,0.1850709279378255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,51200,1.063151995340983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,10240,0.21042346954345703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,8192,0.13533652623494466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,10240,0.2117589314778646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,8192,0.1886090596516927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,7168,0.11823466618855794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,7168,0.1506399949391683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,6144,0.10253439744313557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,6144,0.13047466278076172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,5120,0.0866592009862264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,5120,0.11185599962870281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,4096,0.06960639953613282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2560,256,0.018768000602722167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,4096,0.09173866907755533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,3584,0.061300265789031985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,3584,0.08165333271026612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,16384,0.3190592130025228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,12288,0.24700160026550294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,3072,0.054030934969584145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,3072,0.07245653470357259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,8192,0.15988373756408691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,7168,0.14655466079711915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,2560,0.0449504017829895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,2560,0.06141333182652792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,5120,0.10574933687845867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,6144,0.12711040178934735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,2048,0.05078719854354859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,1536,0.0284223993619283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,1536,0.04050240119298299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,4096,0.08751253286997476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,1024,0.020510933796564736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,1024,0.031769599517186484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,1024,0.02723306616147359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,768,0.016938666502634682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,3584,0.07379840215047201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,768,0.027769599358240766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,512,0.012827733159065246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,512,0.024667733907699586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,2048,0.036652799447377524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,256,0.009302399555842082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,256,0.02165973385175069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,128,0.007238399982452392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,128,0.019913599888483683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,3072,0.06369173526763916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,64,0.00653653343518575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,64,0.01984000007311503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,2048,32,0.006878933310508728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,2048,32,0.019636267423629762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,2560,0.054637865225474036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,1536,0.035368533929189046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,2048,0.04503466685612996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,65536,0.784289042154948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,51200,0.5847359975179036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,768,0.02214399973551432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,65536,1.106757354736328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,512,0.017784533898035686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,16384,0.18686933517456056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,12288,0.14146666526794432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,16384,0.28951679865519203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,12288,0.22092480659484864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,256,0.014845866958300272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,51200,0.9350048065185547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,12288,0.2071850617726644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,10240,0.11865706443786621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,8192,0.09575680096944174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,10240,0.18663360277811686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,8192,0.15291520754496257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,7168,0.08386027018229167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,7168,0.13606826464335126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,6144,0.07317226727803548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,6144,0.11926399866739909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,6144,0.10439893404642742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,2048,128,0.012577066818873087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,5120,0.06180266539255778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,5120,0.10052159627278645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,4096,0.05034666856129964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,4096,0.08158720334370931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,4096,0.06955946286519368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,3584,0.04463040033976237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,3584,0.07157759666442871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,65536,1.0298922856648765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,3072,0.04071466525395711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,16384,0.2530325412750244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,3072,0.051565865675608315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,3072,0.06678187052408854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,2560,0.03285119930903117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,2560,0.056594133377075195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,2048,0.0295306662718455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,2048,0.04673706690470378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,51200,0.8193973541259766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,10240,0.1727413336435954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,2048,0.0404693325360616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,1536,0.023746132850646973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,1536,0.03715946674346924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,1024,0.017498666048049928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,8192,0.14001599947611493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,1024,0.028495999177296956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,7168,0.12023253440856933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,768,0.013347199559211731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,512,0.010620799660682679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,768,0.01879040002822876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,512,0.01511146624883016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,512,0.022107734282811483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,256,0.007923200229803721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,256,0.020064000288645426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,256,0.01260693371295929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,128,0.006596266726652781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,128,0.01853760083516439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,128,0.010818133751551311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,64,0.005941333373387655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1536,32,0.006118399898211161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,64,0.01840426723162333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,32,0.01837546626726786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,5120,0.08639360268910726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,65536,0.5386293411254883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,51200,0.45000534057617186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1536,768,0.0253440002600352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,65536,0.9936661402384439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,3584,0.059765334924062094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,16384,0.1335968017578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,16384,0.2590954621632894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,51200,0.8074517567952475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,12288,0.10004266897837322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,2560,0.045655465126037596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,12288,0.19894293149312336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,10240,0.08398293654123942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,1536,0.02940373420715332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,10240,0.16766079266866046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,8192,0.06776959896087646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1536,1024,0.022858667373657226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,8192,0.13759786287943523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,8192,0.11338133017222088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,7168,0.05922880172729492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,7168,0.12349867026011149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,7168,0.10163839658101399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,6144,0.052317865689595545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,6144,0.10757866700490315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,5120,0.04336106777191162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,5120,0.0908074696858724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,65536,0.8766784032185873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,4096,0.07674453258514405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,51200,0.655193583170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,3584,0.031627732515335086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,16384,0.21583040555318198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,3584,0.05144319931666056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,3584,0.063864533106486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,3072,0.027435733874638872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,3072,0.055029332637786865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,2560,0.02328746716181437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,2560,0.045575467745463054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,10240,0.14025600751241046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,4096,0.03634986480077108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,2048,0.01928960084915161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,2048,0.038203732172648115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,1536,0.01595626672108968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,1536,0.031541333595911665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,6144,0.08409919738769531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,1024,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,1024,0.02590506672859192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,5120,0.07299306392669677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,1024,0.01853333314259847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,768,0.009797333677609762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,768,0.023107200860977173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,512,0.008028799792130788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,512,0.020693333943684895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,4096,0.057911467552185056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,256,0.006293333570162455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,256,0.01853440006573995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,12288,0.16330560048421222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,128,0.005397333204746247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,3072,0.0445525328318278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,128,0.01798080007235209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,64,0.00488319993019104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,2560,0.037717334429423016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,1024,32,0.005257600049177805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,32,0.017478400468826295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,2048,0.030481066306432086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,65536,0.4035530726114909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,1536,0.024406399329503378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,51200,0.32509546279907225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,65536,0.9489664077758789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,768,0.015915733575820924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,1024,64,0.01760960022608439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,16384,0.12608106931050617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,512,0.012549333771069846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,16384,0.24448533058166505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,51200,0.7334346771240234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,12288,0.09728000164031983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,10240,0.08140693505605062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,12288,0.18765226999918622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,10240,0.15933225949605306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,8192,0.06050026814142863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,8192,0.13097813129425048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,256,0.010197333494822184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,7168,0.0565280000368754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,7168,0.12140159606933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,1024,128,0.008667733271916707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,6144,0.04991573492685954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,6144,0.10656320254007975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,65536,0.8156266530354819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,5120,0.04270293315251668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,5120,0.09025386969248453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,16384,0.20995732943216958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,4096,0.033956265449523924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,12288,0.1583029270172119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,4096,0.07050773302714029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,10240,0.1326933304468791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,3584,0.030264532566070555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,3584,0.060950398445129395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,3072,0.026738133033116656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,3072,0.05212053457895914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,8192,0.10917226473490398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,2560,0.022179200251897176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,2560,0.043271466096242266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,7168,0.09218239784240723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,2048,0.018659200270970663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,2048,0.036507733662923175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,51200,0.6275178909301757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,6144,0.08116052945454916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,1536,0.01463573376337687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,1536,0.030458666880925495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,5120,0.06715946992238363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,1024,0.01116373340288798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,1024,0.025412267446517943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,4096,0.05610239903132121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,768,0.00935040016969045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,768,0.022581332921981813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,3584,0.049858132998148605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,512,0.00759680022795995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,512,0.02086826761563619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,3072,0.042513068517049155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,256,0.005644799768924713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,256,0.01837653319040934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,2560,0.036099199453989664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,128,0.004880000154177348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,128,0.01702186663945516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,2048,0.02947733402252197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,64,0.00446720023949941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,64,0.017051732540130614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,1536,0.022731733322143555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,768,32,0.004553600152333578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,768,32,0.017029333114624023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,65536,0.2787007967631022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,51200,0.22106666564941407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,65536,0.8799701054890952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,1024,0.017037866512934367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,16384,0.0715231974919637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,768,0.014434132973353067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,16384,0.23066239356994628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,12288,0.054330666859944664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,512,0.011306666334470113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,12288,0.17848639488220214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,10240,0.05434666474660238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,256,0.009066667159398396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,10240,0.15178240140279134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,8192,0.04200746615727742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,8192,0.12320213317871094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,51200,0.7097930908203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,7168,0.0376362681388855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,768,128,0.007549866537253062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,7168,0.10996159712473552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,6144,0.03288960059483846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,6144,0.09681493441263835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,65536,0.5883029301961262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,5120,0.02866026759147644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,5120,0.0801589330037435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,51200,0.4642730712890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,4096,0.02217493255933126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,4096,0.0637183984120687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,16384,0.16100160280863446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,3584,0.020351999998092653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,3584,0.05417173306147257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,12288,0.1231338659922282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,3072,0.018202666441599527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,3072,0.045006934801737467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,10240,0.10085866451263428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,2560,0.013938132921854654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,2560,0.038016001383463546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,8192,0.08239999612172445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,2048,0.011762133240699768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,2048,0.030641067028045654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,7168,0.07299306392669677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,1536,0.009523199995358785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,6144,0.0616703987121582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,1536,0.02693866689999898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,1024,0.0073077330986658735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,1024,0.02396906614303589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,5120,0.05312853256861368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,768,0.006623999774456024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,768,0.02114773392677307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,4096,0.04445013205210368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,512,0.005610666672388713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,512,0.018976000944773357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,3584,0.03887893358866374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,256,0.004668800036112467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,256,0.017406932512919106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,256,0.008214400211970011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,3072,0.03402560154596965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,128,0.004093866546948751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,128,0.016238933801651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,64,0.003786666691303253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,64,0.015819733341534935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,2560,0.027829333146413164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,512,32,0.004002133260170618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,512,32,0.016275200247764587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,65536,0.17412800788879396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,1536,0.018833067019780478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,65536,0.8292778650919596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,51200,0.13976747194925945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,1024,0.014404267072677612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,16384,0.04620266755421956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,16384,0.21794773737589518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,51200,0.6550559997558594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,768,0.012020267049471537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,12288,0.036429866154988604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,12288,0.16805866559346516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,10240,0.03132479985555013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,512,0.009876267115275065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,10240,0.1424437363942464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,128,0.006808533271153767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,8192,0.02561066746711731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,512,2048,0.02364586591720581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,8192,0.11584959824879963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,8192,0.07548906803131103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,7168,0.023362133900324503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,7168,0.10284799734751385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,6144,0.019526400168736777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,6144,0.08865386644999186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,65536,0.5615093231201171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,5120,0.014382933576901754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,5120,0.07602346738179525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,51200,0.4332714716593425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,4096,0.01195093293984731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,4096,0.05921813249588013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,16384,0.14423999786376954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,3584,0.01081706682840983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,12288,0.10776533285776775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,3584,0.05008213520050049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,3072,0.00986346701780955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,3072,0.04137920141220093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,2560,0.011454932888348897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,10240,0.09158399899800619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,2560,0.03483733336130778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,2048,0.009730133414268493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,2048,0.02993280092875163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,7168,0.0667306661605835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,1536,0.008021333316961924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,1536,0.025276799996693928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,6144,0.05697706540425619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,1024,0.00628053347269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,1024,0.022233599424362184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,5120,0.050057601928710935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,768,0.0056874667604764305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,768,0.02004800041516622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,4096,0.04018346468607585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,512,0.004693333307902018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,512,0.018032000462214152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,3584,0.03547733227411906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,256,0.0037834666669368743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,256,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,3072,0.030177066723505657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,128,0.0034111998975276947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,128,0.015595733126004537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,2560,0.025617067019144697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,64,0.003221333275238673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,64,0.01605226695537567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,256,32,0.003443199892838796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,256,32,0.0156960000594457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,2048,0.021543467044830324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,65536,0.15873707135518392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,1536,0.01686720053354899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,65536,0.4431925455729167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,51200,0.12736533482869467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,65536,0.8165023803710938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,16384,0.046801066398620604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,1024,0.01300373375415802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,16384,0.11841066678365071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,16384,0.21532692909240722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,12288,0.036855467160542804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,51200,0.6408992131551107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,12288,0.09048213164011637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,10240,0.03158933321634928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,12288,0.1661237398783366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,10240,0.07506986459096274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,8192,0.02681279977162679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,10240,0.1402239958445231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,8192,0.06055146853129069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,7168,0.020891733964284263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,768,0.011012267072995503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,8192,0.11364373366038005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,6144,0.014733866850535075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,7168,0.05427093505859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,7168,0.10138453642527263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,6144,0.08668266932169597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,5120,0.07255893548329671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,5120,0.03938773473103841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,4096,0.010734933614730834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,4096,0.054202667872111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,3584,0.009733333190282186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,512,0.008703999718030294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,3584,0.04586026668548584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,3072,0.008724266290664673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,3072,0.038381866614023846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,256,0.007226666808128357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,2560,0.007740800082683563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,2560,0.03287893335024516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,256,128,0.005997866888840993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,2048,0.007029333213965099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,2048,0.0295360008875529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,5120,0.012370133399963379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,1536,0.006040533383687338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,1536,0.02544959982236226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,1536,0.01493119994799296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,1024,0.005095466474692027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,1024,0.021540266275405884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,1024,0.011606400211652119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,768,0.004570666452248891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,51200,0.34852587381998695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,768,0.01949119965235392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,768,0.010007466872533162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,512,0.003976533313592275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,512,0.018326399723688762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,6144,0.04673813184102376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,256,0.003619199991226196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,256,0.016713599363962807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,256,0.007257600128650665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,4096,0.032519465684890746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,128,0.0032672000428040824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,128,0.006146133442719777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,128,0.015607466300328573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,64,0.0030720000465710956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,128,32,0.0032373333970705668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,64,0.015347199638684592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,3584,0.02892799973487854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,128,32,0.015219199657440185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,3072,0.02476159930229187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,65536,0.15016427040100097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,51200,0.11770880222320557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,16384,0.042949334780375166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,12288,0.033868801593780515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,2560,0.021331199010213218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,16384,0.21452372868855796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,10240,0.02898453275362651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,12288,0.16412159601847331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,8192,0.02403200070063273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,7168,0.0192138671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,10240,0.13853972752888996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,8192,0.11300480365753174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,6144,0.012958932916323343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,5120,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,7168,0.09966506958007812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,65536,0.8118026733398438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,6144,0.08639466762542725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,2048,0.017966934045155845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,5120,0.07171200116475424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,3584,0.008437333504358928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,3072,0.008319999774297078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,3584,0.045380266507466634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,4096,0.05424426794052124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,2560,0.0072512000799179075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,3072,0.037869866689046225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,2048,0.006140799820423126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,1536,0.005367466807365417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,2048,0.028828799724578857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,1024,0.004445866743723551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,1536,0.024606933196385704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,768,0.0041354666153589886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,1024,0.021396267414093017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,512,0.003731200098991394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,768,0.019669334093729653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,256,0.003289599965016047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,512,0.017892267306645712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,128,0.0030122667551040648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,256,0.01622719963391622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,64,0.0029557332396507262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,128,0.015729066729545594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,32,0.003014400104681651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,64,0.015227733055750528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,32,0.015102932850519816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,65536,0.15042880376180012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,51200,0.12550613085428874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,16384,0.044240001837412515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,16384,0.21446186701456704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,12288,0.0333024005095164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,10240,0.027784534295399982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,12288,0.16399466196695964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,51200,0.6382357279459636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,51200,0.6375050862630208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,8192,0.024210133155186973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,64,4096,0.009621333082516987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,7168,0.02062079906463623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,10240,0.1378335952758789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3073,128,512,0.008411733309427898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,5120,0.011370666821797689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,7168,0.09999039967854818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,8192,0.11297386487325031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,4096,0.010587732990582783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,6144,0.08604053656260172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,3584,0.009739733735720317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,5120,0.07074346542358398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,64,2560,0.03278506596883138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,3072,0.008388266960779826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,4096,0.0538538654645284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,3584,0.04503466685612996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,2048,0.006248533229033152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,3072,0.037445334593455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,2560,0.032151466608047484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,2048,0.0287989338239034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,1024,0.004375466704368591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,1536,0.02539520064989726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,768,0.003962666789690653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,1024,0.022395733992258707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,512,0.0035829332967599234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,768,0.019323732455571493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,256,0.003218133250872294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,512,0.018333866198857626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,128,0.0029311999678611755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,256,0.01602453291416168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,64,0.002868266652027766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,128,0.015576533476511636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,32,0.002867199977238973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,64,0.015236266454060874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,32,0.014884266257286071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3073,32,65536,0.8140074412027994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,6144,0.013369599978129068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,2560,0.007541333138942718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3073,32,1536,0.0051360001166661584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,16384,4.895465596516927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,12288,3.476968638102213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,16384,5.832896931966146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,12288,6.6089324951171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,16384,8.712364705403646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,12288,4.316549173990885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,10240,3.281777191162109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,10240,3.7218058268229166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,10240,5.464669799804687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,8192,2.4042388916015627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,8192,3.144927978515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,8192,5.238684590657552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,7168,2.091814422607422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,51200,16.011174519856773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,7168,4.652152506510417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,6144,2.002959950764974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,6144,3.7944480895996096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,5120,1.5261632283528646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,5120,1.837660853068034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,5120,2.8290667215983074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,7168,2.615461222330729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,4096,1.226468276977539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,3584,1.0626933415730795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,4096,2.2750783284505207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,4096,1.5444352467854818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,51200,21.01716105143229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,6144,2.254719034830729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,3584,1.3879093170166015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,3584,1.9516426086425782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,3072,0.9733354568481445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,2560,0.7478005091349285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,3072,1.1958783467610679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,3072,1.7759019215901692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,2560,1.4458666483561198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,2048,1.1398612976074218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,2048,0.6782186508178711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,2048,0.9009237289428711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,1536,0.49504639307657883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,1536,0.9048224131266276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,1024,0.4142719904581706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,1024,0.6177162806193034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,768,0.3052106539408366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,1024,0.5189717292785645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,768,0.46143360137939454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,512,0.34598185221354166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,768,0.48074026107788087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,256,0.2142186641693115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,256,0.19161814053853352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,512,0.36303574244181314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,128,0.17272000312805175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,256,0.31057494481404624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,128,0.17363306681315105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,64,0.1618666648864746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,128,0.2823871930440267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,64,0.1674634615580241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,32,0.15560426712036132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,32,0.16852480570475262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,65536,512,0.23196266492207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,2560,0.9092330932617188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,65536,1536,0.6321269353230794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,65536,51200,28.403041585286456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,16384,3.834715779622396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,16384,6.882711283365886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,51200,11.771578979492187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,16384,4.465056864420573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,12288,5.360669962565104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,65536,16.307811482747397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,12288,3.315931701660156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,51200,17.14820556640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,10240,2.5744064331054686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,10240,4.63387196858724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,10240,3.040119425455729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,12288,3.4770070393880212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,8192,3.485581970214844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,8192,2.0707018534342447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,8192,2.2681620279947916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,7168,1.6534496307373048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,65536,22.880179850260415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,51200,22.764162190755208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,7168,3.047262827555339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,7168,1.9811487833658856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,6144,1.5004629770914713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,6144,1.6937515258789062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,5120,1.2154399871826171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,6144,2.689844258626302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,5120,1.3949258168538412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,4096,0.9812917073567708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,5120,2.1824811299641924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,4096,1.1770559946695962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,4096,1.775090154012044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,3584,0.8502954483032227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,3584,1.0396629333496095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,3072,0.7375317255655924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,3584,1.6072352091471356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,2560,0.6285194396972656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,3072,0.9114421208699545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,3072,1.3881034851074219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,2560,1.1583700815836588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,2560,0.7703680038452149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,2048,0.5189226786295573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,2048,0.9172192255655924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,2048,0.6988362630208333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,1536,0.7004202524820964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,1536,0.4594069480895996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,1024,0.519648011525472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,1024,0.3266421318054199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,1536,0.5540085474650065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,768,0.38061866760253904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,1024,0.4119562784830729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,768,0.2590986728668213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,512,0.26418026288350427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,512,0.2072981357574463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,256,0.16782719294230145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,256,0.15323947270711263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,768,0.46094932556152346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,512,0.3035008112589518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,128,0.13829013506571453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,256,0.24698987007141113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,128,0.14181866645812988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,64,0.14073386192321777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,51200,128,0.2366410732269287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,51200,32,0.13649600346883137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,65536,28.435548909505208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,64,0.13143786589304607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,51200,32,0.12682027022043865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,16384,1.3366026560465494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,16384,2.2973696390787763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,16384,1.5327733357747397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,51200,4.110961151123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,12288,1.7196756998697917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,51200,4.596133422851563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,12288,1.0091775894165038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,65536,5.526681518554687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,10240,0.842466163635254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,10240,0.9542666753133139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,65536,6.7052958170572925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,10240,1.397964859008789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,8192,1.1293076833089193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,8192,0.6868010838826497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,51200,7.417085774739583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,7168,0.603772799173991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,7168,0.6656341552734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,7168,1.0309109369913736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,6144,0.5190944035847982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,6144,0.5511050542195638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,6144,0.8492810567220053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,5120,0.728883171081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,4096,0.3541440010070801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,4096,0.5551829020182292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,4096,0.40243199666341145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,3584,0.4799296061197917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,12288,1.0432042439778646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,65536,9.202231852213542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,3584,0.35560534795125326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,3072,0.3029088020324707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,3072,0.43868160247802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,2560,0.3594474792480469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,5120,0.46639359792073565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,2560,0.25194026629130045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,2048,0.33249279657999675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,2048,0.19005866050720216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,2048,0.24248000780741372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,1536,0.24644053777058922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,1536,0.15294186274210614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,1024,0.16979626019795735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,8192,0.707748285929362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,3584,0.3227381388346354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,1024,0.16266026496887206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,768,0.13828907012939454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,768,0.0955189307530721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,768,0.13125759760538738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,512,0.10159893035888672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,512,0.07564799785614014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,256,0.06461120049158732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,3072,0.3022357304890951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,128,0.05251199801762899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,256,0.08244053522745767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,128,0.05831253528594971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,128,0.07622719605763753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,64,0.04261226654052734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,64,0.059411199887593594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,32,0.04431999921798706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,5120,0.4461781183878581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,32,0.059388800462087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,16384,1024,0.11497279802958171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,16384,256,0.06338560183842977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,2560,0.2438741366068522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,1536,0.17336319287618002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,16384,512,0.09937280019124349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,16384,1.7427349090576172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,16384,0.9867381413777669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,51200,3.208424631754557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,65536,4.140389251708984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,65536,4.671770731608072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,16384,1.1649876912434896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,51200,3.5522613525390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,12288,0.7496501286824544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,12288,1.227791976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,51200,5.197591654459635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,12288,0.8347861607869467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,10240,0.6316672007242838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,10240,0.6898730595906575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,10240,1.0640415827433267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,8192,0.5278197288513183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,8192,0.8149514516194662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,8192,0.5689151763916016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,65536,6.59940694173177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,7168,0.4826218605041504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,7168,0.6798346837361653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,7168,0.507968012491862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,6144,0.6032032012939453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,6144,0.42378028233846027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,5120,0.3325258572896322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,5120,0.3800405184427897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,5120,0.5168821334838867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,4096,0.43156906763712566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,4096,0.2987071990966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,3584,0.2464757283528646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,3584,0.34667838414510094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,3584,0.2721994717915853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,3072,0.29405333201090494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,3072,0.225927464167277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,2560,0.25578667322794596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,2560,0.19198932647705078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,2048,0.2009215990702311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,2560,0.20779733657836913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,2048,0.1642805258433024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,1536,0.16150719324747723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,1536,0.11736000378926595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,1024,0.11538453102111816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,1024,0.09569066365559896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,768,0.09091626803080241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,1024,0.11230719884236653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,768,0.07773013114929199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,512,0.06656426588694254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,768,0.0981813351313273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,6144,0.4137440045674642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,256,0.04506453275680542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,512,0.07962133089701334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,128,0.036660265922546384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,256,0.0516981323560079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,256,0.06557866732279459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,64,0.03193599979082744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,128,0.04810880025227864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,128,0.05923733313878378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,12288,32,0.03341546853383382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,64,0.047416532039642335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,32,0.04782400131225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,12288,512,0.06394133170445761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,4096,0.27943894068400066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,3072,0.22195520401000976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,2048,0.1604885260264079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,12288,1536,0.1313258647918701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,16384,0.8665781021118164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,65536,3.5885492960611978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,16384,1.3754837036132812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,65536,3.9330324808756507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,16384,0.9758368174235026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,12288,0.7309312184651693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,12288,1.0382858912150064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,51200,2.8230133056640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,12288,0.6806986490885417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,10240,0.5621525446573894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,8192,0.46966400146484377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,51200,3.096791585286458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,10240,0.5910015741984049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,51200,4.371260579427084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,10240,0.9047744115193685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,8192,0.6907199859619141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,8192,0.4677280108133952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,7168,0.452071475982666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,7168,0.45367040634155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,6144,0.3732768058776855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,7168,0.6165760040283204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,6144,0.3886485417683919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,6144,0.4952981313069661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,5120,0.4502133369445801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,5120,0.31281280517578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,4096,0.2610154628753662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,4096,0.2592362721761068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,4096,0.3526485443115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,3584,0.2959957440694173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,3584,0.2378314654032389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,3072,0.19229226112365722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,3584,0.2302079995473226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,3072,0.2582026640574137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,3072,0.20914667447408042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,2560,0.22800854047139488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,2560,0.17166293462117513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,2560,0.16898986498514812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,2048,0.17902612686157227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,2048,0.14101120630900066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,2048,0.14846080144246418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,1536,0.14560106595357258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,1536,0.11146240234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,1024,0.10282879670461018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,1024,0.08573973178863525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,1024,0.11065813700358074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,768,0.07210026582082113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,512,0.06381760040918985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,768,0.08089386622111003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,256,0.04014613231023152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,512,0.05983359813690185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,512,0.0756277322769165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,128,0.032330665985743204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,256,0.04739733139673869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,256,0.05801920096079508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,64,0.02951040069262187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,128,0.0424778660138448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,128,0.051151998837788905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,32,0.029918932914733888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,64,0.042565333843231204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,10240,32,0.042701868216196696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,65536,5.480460611979167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,10240,768,0.08052053451538085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,5120,0.29252265294392904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,10240,1536,0.11174293359120686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,65536,2.933856964111328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,51200,2.311511484781901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,65536,3.231698099772135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,16384,0.7301237106323242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,16384,1.1018656412760417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,16384,0.756386121114095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,12288,0.5979775746663412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,12288,0.8229610443115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,51200,3.566681671142578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,12288,0.5729589462280273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,10240,0.713595708211263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,65536,4.370222981770834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,51200,2.4622347513834635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,10240,0.4950474739074707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,10240,0.46855465571085614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,8192,0.4100202560424805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,8192,0.5590944290161133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,8192,0.3905440012613932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,7168,0.34419094721476234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,6144,0.4035039901733398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,7168,0.37453654607137044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,7168,0.5059413274129232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,6144,0.30167786280314124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,5120,0.24009706179300944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,5120,0.3323338508605957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,5120,0.24110719362894692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,4096,0.210426664352417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,3584,0.23832000096639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,3584,0.17197759946187335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,3072,0.200601593653361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,3072,0.16185706456502277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,2560,0.16613653500874836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,2560,0.13522879282633465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,2560,0.14122559229532877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,2048,0.13896320660909017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,2048,0.11532266934712727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,1536,0.10942613283793132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,2048,0.11796480019887287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,1024,0.0762762705485026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,1536,0.09165333112080892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,1536,0.10224106311798095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,1024,0.07175573507944742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,768,0.06459199984868368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,1024,0.07522346973419189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,512,0.0460426648457845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,768,0.06077760060628255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,512,0.04912319978078206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,512,0.055027198791503903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,256,0.029785599311192828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,256,0.041117866834004715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,128,0.025265065828959148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,128,0.03754986524581909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,4096,0.21492053667704264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,4096,0.28495359420776367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,128,0.04157013495763143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,64,0.023156267404556275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,8192,32,0.024168533086776734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,64,0.038867199420928956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,32,0.038330666224161786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,8192,3072,0.15702932675679523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,768,0.07081387042999268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,6144,0.2810218811035156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,3584,0.171726926167806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,16384,0.9958815892537436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,65536,2.700103505452474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,8192,256,0.04456319808959961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,51200,2.1118624369303385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,51200,3.2193557739257814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,65536,3.4334388732910157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,16384,0.7400149027506511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,12288,0.5266005198160808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,12288,0.733022944132487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,65536,4.06869862874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,10240,0.44927574793497727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,8192,0.5227189381917318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,10240,0.6550581614176433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,8192,0.35448532104492186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,7168,0.4571093241373698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,7168,0.3137322743733724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,6144,0.36001599629720055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,6144,0.2770538647969564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,16384,0.7426549275716146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,12288,0.5722805023193359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,5120,0.3023541450500488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,10240,0.5083765347798666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,5120,0.2371573289235433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,51200,2.630845896402995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,4096,0.2488032023111979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,4096,0.19162559509277344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,8192,0.38626238505045574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,3584,0.2203690687815348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,3584,0.1707562605539958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,3584,0.186627197265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,3072,0.19182186126708983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,3072,0.15004587173461914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,7168,0.3491711934407552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,2560,0.1609781265258789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,2560,0.12937493324279786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,2048,0.1289120038350423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,2048,0.11040533383687337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,1536,0.1011082649230957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,1536,0.08898026943206787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,1024,0.07252480189005533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,1024,0.06745386918385823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,6144,0.2876426696777344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,1024,0.06746666431427002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,768,0.05575679937998453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,768,0.05713599920272827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,512,0.0430239995320638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,512,0.04770559867223104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,5120,0.24303359985351564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,256,0.02865920066833496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,4096,0.20259307225545248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,256,0.03806613286336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,128,0.023257599274317423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,128,0.035366400082906084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,3072,0.1473354657491048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,64,0.021126399437586464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,64,0.03588693141937256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,7168,32,0.021461333831151327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,2560,0.13142506281534833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,2048,0.10886720021565754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,1536,0.08246293067932128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,7168,32,0.03554133176803589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,768,0.05185173352559408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,512,0.04180906613667806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,51200,1.8003456115722656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,65536,2.316527048746745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,65536,2.5348981221516924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,51200,2.659271494547526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,51200,1.8767157236735028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,256,0.034828801949818924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,7168,128,0.02957013249397278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,16384,0.5940725326538085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,12288,0.4228927930196126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,16384,0.821894391377767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,12288,0.4930432001749675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,65536,3.3836692810058593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,10240,0.47557226816813153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,16384,0.6205535888671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,10240,0.3821983973185221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,10240,0.3575231870015462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,8192,0.3276426633199056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,8192,0.4532586733500163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,7168,0.33680852254231775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,8192,0.3129642804463705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,7168,0.3127274513244629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,7168,0.2572415987650553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,6144,0.29254827499389646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,5120,0.20444159507751464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,4096,0.18018239339192707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,4096,0.15876800219217937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,3584,0.17644160588582355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,3584,0.14894293149312338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,3584,0.14890880584716798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,12288,0.5838794708251953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,3072,0.1537834644317627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,3072,0.13211733500162762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,2560,0.13277013301849366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,3072,0.11906452973683675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,2560,0.11326506932576497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,2048,0.10453120072682698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,2560,0.11166186332702636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,2048,0.09557653268178304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,1536,0.08316480318705241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,2048,0.08708053429921468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,1536,0.07694400151570638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,1024,0.0576533317565918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,1536,0.07763840357462565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,6144,0.24300053914388023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,5120,0.26280320485432945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,6144,0.25617705980936684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,1024,0.05953280131022135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,5120,0.20468907356262206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,768,0.05027413368225098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,768,0.04998826583226522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,4096,0.20726399421691893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,256,0.024668800830841064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,512,0.04302933216094971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,512,0.04233920176823934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,128,0.019422932465871175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,256,0.03569813172022502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,256,0.03517866532007853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,128,0.031769599517186484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,64,0.01773973306020101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,128,0.03306986689567566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,32,0.018397865692774455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,32,0.03162239988644917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,6144,1024,0.0560479998588562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,768,0.04917226632436116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,6144,512,0.03561173280080159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,6144,64,0.03174186746279399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,65536,2.0476661682128907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,16384,0.6389322916666667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,16384,0.533461316426595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,65536,2.9411966959635416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,51200,1.7059146881103515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,51200,1.9571392059326171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,12288,0.4905695915222168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,51200,2.2712799072265626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,12288,0.40430078506469724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,10240,0.3374506632486979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,10240,0.4454943974812825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,10240,0.39622720082600915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,8192,0.37074880599975585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,8192,0.27621332804361975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,7168,0.3130762736002604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,7168,0.26591893831888835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,6144,0.24476693471272787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,6144,0.22655040423075357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,5120,0.21236586570739746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,5120,0.18861440022786458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,5120,0.20693119366963705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,4096,0.16645867029825848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,4096,0.15111039479573568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,4096,0.154365873336792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,3584,0.1509343942006429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,3584,0.13067626953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,3072,0.12752213478088378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,3072,0.11593600114186604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,65536,2.4138058980305988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,2560,0.13055466810862223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,16384,0.5586474736531575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,2560,0.11115840276082356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,2560,0.10756800174713135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,2048,0.09892799854278564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,2048,0.08795413176218668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,1536,0.07132266362508138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,2048,0.0973418633143107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,12288,0.433191458384196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,1536,0.07234346866607666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,1024,0.05604586601257324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,8192,0.29074878692626954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,7168,0.263862403233846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,768,0.04082346757253011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,768,0.046803200244903566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,6144,0.22217067082722983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,512,0.03092693289120992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,512,0.03935573498408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,256,0.02172266642252604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,256,0.03206506570180257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,1024,0.050748801231384276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,128,0.017335466543833413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,128,0.0291978657245636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,3584,0.1325813372929891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,128,0.023131734132766722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,64,0.014661332964897156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,64,0.029021867116292316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,5120,32,0.014839466412862143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,5120,32,0.02849173347155253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,3072,0.11246399879455567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,1536,0.06288746595382691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,1024,0.0509493350982666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,65536,1.832106653849284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,65536,1.7773653666178386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,65536,2.266679382324219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,768,0.04061439832051595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,512,0.032440533240636185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,5120,256,0.027127466599146527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,16384,0.5543221155802409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,12288,0.38228588104248046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,16384,0.3994677225748698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,12288,0.34365971883138025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,16384,0.49432214101155597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,51200,1.6573579152425129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,12288,0.3361749331156413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,10240,0.3268704096476237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,51200,1.3949930826822916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,10240,0.3007424036661784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,10240,0.24982719421386718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,8192,0.25508480072021483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,7168,0.18665493329366048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,7168,0.24447360038757324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,6144,0.1926655928293864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,7168,0.2199456055959066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,8192,0.26531732877095543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,6144,0.19974932670593262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,6144,0.16216319402058918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,5120,0.1624618689219157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,5120,0.16176640192667643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,4096,0.13561065991719562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,5120,0.14310293197631835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,4096,0.117522136370341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,4096,0.12923093636830646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,3584,0.11938026746114094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,3584,0.10667946338653564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,3584,0.11895466645558675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,3072,0.1036959966023763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,2560,0.09149440129597983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,3072,0.10218133131663006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,2560,0.0891808032989502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,2560,0.0797216018040975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,2048,0.07178026835123698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,2048,0.07649493217468262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,2048,0.06733226776123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,1536,0.05776853164037069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,1536,0.06371946732203165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,1536,0.05436906814575195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,1024,0.049779200553894044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,1024,0.04379200140635173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,768,0.03340160051981608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,512,0.02598186731338501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,768,0.042607998847961424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,512,0.030925865968068438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,512,0.03563946485519409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,256,0.018234666188557944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,256,0.029838933547337847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,256,0.025704532861709595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,128,0.013960533340771995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,128,0.02315733234087626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,64,0.012121599912643433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,8192,0.2257919947306315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,64,0.02622720003128052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,32,0.012718933820724487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,51200,1.3719125111897787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,32,0.02614186604817708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,4096,1024,0.041161600748697916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,4096,128,0.02693866689999898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,65536,1.7165152231852214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,65536,2.0533087412516275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,51200,1.591971206665039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,3072,0.08612266381581625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,16384,0.43472105662028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,51200,1.3460491180419922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,12288,0.3462805430094401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,4096,768,0.03642026583353679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,12288,0.3329375902811686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,10240,0.29909439086914064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,10240,0.2821375846862793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,8192,0.24031999905904136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,8192,0.22574933369954428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,7168,0.21626666386922203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,7168,0.19813119570414225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,16384,0.5067541440327962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,6144,0.1834378719329834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,6144,0.1725279966990153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,12288,0.303330135345459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,5120,0.15221333503723145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,5120,0.14618239402770997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,65536,1.7492373148600262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,4096,0.13124053478240966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,4096,0.12710826396942138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,16384,0.4067359924316406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,3584,0.1066538651784261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,3584,0.10992746353149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,6144,0.15600214004516602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,10240,0.26032533645629885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,3072,0.09549012978871664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,3072,0.09348159631093343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,2560,0.0798314650853475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,2560,0.08112853368123372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,7168,0.1844000021616618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,2048,0.06586666504542033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,2048,0.0688981294631958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,51200,1.364407475789388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,1536,0.054598399003346766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,1536,0.060781868298848477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,8192,0.20650986035664878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,1024,0.035624531904856364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,1024,0.04274773200352987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,3072,0.0824682633082072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,768,0.02841493288675944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,768,0.04033493200937907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,5120,0.13159680366516113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,512,0.021150932709376017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,512,0.0345909317334493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,4096,0.10938026905059814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,256,0.017909334103266398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,256,0.029049599170684816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,2560,0.07228373686472575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,128,0.013726933797200521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,128,0.02690133253733317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,3584,0.09601919651031494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,1536,0.047380268573760986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,64,0.011901866396268208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,64,0.025886933008829754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3584,32,0.012495999534924824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3584,32,0.025731199979782106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,1024,0.037910401821136475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,2048,0.05955946842829386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,65536,1.5911242167154946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,65536,1.4840202331542969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,65536,1.64171625773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,51200,1.2638954162597655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,16384,0.37322880427042643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,16384,0.4283882776896159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,12288,0.2976597468058268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,12288,0.3187573432922363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,128,0.01662826637427012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,10240,0.23749866485595703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,51200,1.1707349141438803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,768,0.030055467287699384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,10240,0.24694933891296386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,10240,0.2530282656351725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,8192,0.18830933570861816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,512,0.024216532707214355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,8192,0.20086399714152017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,7168,0.16620480219523112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,8192,0.2010741392771403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,7168,0.1760800043741862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,6144,0.14602986971537274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,6144,0.15434026718139648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,6144,0.15348374048868815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,5120,0.12085653146107991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,5120,0.1302986701329549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,4096,0.10009706815083821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,4096,0.10703252951304118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,4096,0.10786453088124592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,3584,0.08658239841461182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,3584,0.10121920108795165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,3584,0.09393813610076904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,3072,0.07727039655049642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3584,256,0.01983786622683207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,2560,0.06396053234736124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,3072,0.08413333098093669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,2560,0.0731381336847941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,2048,0.052393599351247155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,2048,0.06148800055185953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,16384,0.38967361450195315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,51200,1.2035040537516275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,1536,0.04067306518554688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,1536,0.046513064702351885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,1536,0.05065493186314901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,1024,0.030964267253875733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,1024,0.03979733387629191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,1024,0.036830933888753255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,768,0.02611733277638753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,768,0.03401813507080078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,512,0.01876693367958069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,512,0.02912213404973348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,12288,0.2954517364501953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,256,0.013148799538612366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,256,0.024139734109242757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,7168,0.17694826126098634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,128,0.009939199686050415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,128,0.022016000747680665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,5120,0.12610026995340984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,64,0.008433066805203756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,64,0.021356799205144248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,3072,32,0.008748799562454224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,3072,32,0.02148373325665792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,3072,0.0786677360534668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,2560,0.06929173469543456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,2048,0.05795413255691528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,65536,1.2957450866699218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,768,0.029028266668319702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,512,0.0231605331103007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,65536,1.3832511901855469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,51200,1.0655733108520509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,16384,0.32064425150553383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,256,0.018942934274673463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,16384,0.3588970820109049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,12288,0.24240214029947915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,12288,0.2773962656656901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,12288,0.2531744003295898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,51200,1.0977099100748697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,16384,0.36846720377604164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,10240,0.21359786987304688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,8192,0.16238187154134115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,10240,0.23452266057332358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,8192,0.20292906761169432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,7168,0.1468554655710856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,7168,0.16850560506184895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,6144,0.1274901310602824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,3072,128,0.01565333306789398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,5120,0.10484053293863933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,6144,0.14642346700032552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,5120,0.12217706839243572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,4096,0.08520320256551107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,4096,0.10088853041330974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,65536,1.4138442993164062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,3584,0.08552959760030111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,3584,0.09534613291422525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,10240,0.22213865915934244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,51200,1.0997173309326171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,8192,0.1781450589497884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,3072,0.0715231974919637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,7168,0.15688212712605792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,3072,0.07930880387624105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,2560,0.05559253295262655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,6144,0.13231680393218995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,2560,0.06866986751556396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,2048,0.04540266593297322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,5120,0.11477546691894532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,2048,0.057749334971110025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,1536,0.0361130674680074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,1536,0.04619306723276774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,1024,0.02579200069109599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,1024,0.036451200644175216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,4096,0.09311359723409017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,768,0.021257599194844566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,768,0.031141332785288495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,2560,0.06258133252461752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,512,0.016350932916005454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,512,0.02624746759732564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,3584,0.08090879917144775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,256,0.011252267162005107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,256,0.02399466633796692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,128,0.010487467050552368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,3072,0.06967360178629557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,128,0.022606933116912843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,64,0.009551999966303508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,64,0.02250986695289612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2560,32,0.010046933094660442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,2048,0.05037333170572916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2560,32,0.022900267442067464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,1024,0.030822400252024335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,768,0.02476373314857483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,65536,1.1208138783772787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,65536,1.2585759480794272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,65536,0.9476437250773111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,1536,0.03834773302078247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,51200,0.8214133580525717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,512,0.01933120091756185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,16384,0.27502291997273765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,16384,0.32844479878743493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,12288,0.19943040211995441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,16384,0.29842987060546877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,12288,0.2705024083455404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,12288,0.18663573265075684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,10240,0.16734293301900227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,51200,1.0156021118164062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,10240,0.23625493049621582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,10240,0.17047573725382487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,8192,0.16895359357198078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,8192,0.13685866991678874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,7168,0.11890133221944173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,7168,0.15508480072021485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,7168,0.11250560283660889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,6144,0.13860586484273274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,5120,0.08612373669942221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,6144,0.10268692970275879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,4096,0.06911466916402181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,5120,0.1147114674250285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,256,0.016223999857902526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,4096,0.06840213139851889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,4096,0.09299519856770834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,3584,0.06056106487909953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,3584,0.06119466622670492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2560,128,0.013645866513252258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,3584,0.08255893389383952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,2560,0.04487359921137492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,3072,0.07194240093231201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,2560,0.061861332257588705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,2048,0.036507733662923175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,2048,0.051335465908050534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,2048,0.04039573272069295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,1536,0.02826133370399475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,1536,0.041050668557484946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,8192,0.1362730662027995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,1536,0.032739200194676715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,1024,0.021125332514444987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,1024,0.03213013410568237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,768,0.01697173317273458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,6144,0.10022506713867188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,768,0.028258132934570312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,768,0.02225493391354879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,512,0.013223466277122498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,512,0.024156800905863442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,256,0.009416533509890239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,256,0.020950400829315187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,256,0.015495466192563376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,128,0.007477333148320516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,128,0.01959786613782247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,3072,0.05323839982350668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,64,0.0064650664726893115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,64,0.019511467218399046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,32,0.006833066542943318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,2048,32,0.019453867276509603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,51200,0.7537610371907552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,65536,0.745302391052246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,2048,3072,0.05248106718063354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,5120,0.08176533381144205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,65536,1.1099722544352213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,2560,0.04659306605656942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,51200,0.5778720219930013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,16384,0.18778880437215167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,1024,0.0258026659488678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,16384,0.3018656094868978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,12288,0.14345919291178386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,51200,0.8646325429280599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,512,0.018428800503412883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,10240,0.11951253414154053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,12288,0.23024427096048988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,2048,128,0.013777066270510355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,10240,0.18759786287943522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,8192,0.09452266693115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,8192,0.1533674716949463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,7168,0.08541759649912516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,7168,0.1361514727274577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,6144,0.0732917308807373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,6144,0.11894079844156902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,12288,0.16945172945658366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,5120,0.06133866707483927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,5120,0.10078826745351155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,65536,0.9079893112182618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,4096,0.0540832002957662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,4096,0.08570666313171386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,16384,0.22728853225708007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,3584,0.045187199115753175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,3584,0.07285866737365723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,10240,0.14338453610738117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,3072,0.03864106734593709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,3072,0.06266453266143798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,51200,0.6944746653238932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,7168,0.10325866540273029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,2560,0.03266879916191101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,2560,0.05311146577199301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,2048,0.02696320017178853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,2048,0.04398080110549927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,6144,0.0874837319056193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,1536,0.02153600056966146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,1536,0.03556586503982544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,8192,0.11700479984283448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,1024,0.0160778671503067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,1024,0.029418667157491047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,3072,0.048086400826772055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,768,0.013173333803812661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,768,0.024757333596547446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,4096,0.06127253373463949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,512,0.01053546667098999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,512,0.02241706649462382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,5120,0.07546026706695556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,256,0.007815466821193695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,1536,0.02755519946416219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,256,0.020038400093714395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,128,0.006498133142789205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,128,0.018590933084487914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,3584,0.05614506800969442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,64,0.005838933090368906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,64,0.018332799275716148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1536,32,0.006004266440868378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1536,32,0.01834133267402649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,2560,0.04241173267364502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,65536,0.5358922958374024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,1024,0.02184213399887085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,51200,0.4005386670430501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,2048,0.03489493529001872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,65536,0.9942090352376303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,65536,0.8931658426920572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,16384,0.13218773206075032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,16384,0.2681173324584961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,12288,0.10260266462961834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,51200,0.7749792098999023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,12288,0.2023146629333496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,10240,0.08608213265736898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,10240,0.169977601369222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,128,0.010754133264223736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,8192,0.0667744000752767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,8192,0.1367967923482259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,256,0.012727466225624085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,512,0.015449600418408713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,7168,0.05981013377507528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,7168,0.12267733414967855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,16384,0.2158730665842692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,6144,0.05164586702982584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,6144,0.1071168025334676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,51200,0.6611914952596029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,5120,0.04332799911499023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,5120,0.09080959955851237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,12288,0.16319146156311035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,4096,0.03559039831161499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1536,768,0.01812053322792053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,4096,0.07313386599222818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,3584,0.031268266836802165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,3584,0.0637226661046346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,3072,0.026954666773478193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,10240,0.13750720024108887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,3072,0.05531520048777262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,2560,0.02302186687787374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,2560,0.04526826540629069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,8192,0.11339946587880452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,2048,0.019250132640202842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,7168,0.09851733048756918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,2048,0.03806399901707967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,6144,0.08419520060221354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,1536,0.015589333573977151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,1536,0.03107306758562724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,5120,0.07029120127360025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,1024,0.01158186693986257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,1024,0.025305600961049397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,4096,0.0586570660273234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,1024,0.018078933159510292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,768,0.00986240009466807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,768,0.023150932788848878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,3584,0.05130453507105509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,512,0.008008533219496409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,3072,0.04456640084584554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,256,0.006267733375231425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,512,0.020722132921218873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,256,0.018577067057291667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,128,0.0053162669142087305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,2560,0.03840959866841634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,128,0.017170133193333943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,128,0.008540800213813782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,64,0.017249067624409996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,32,0.004961066444714864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,1024,32,0.016988799969355265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,65536,0.3842346509297689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,1024,64,0.00477866679430008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,65536,0.9472885131835938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,51200,0.3044981320699056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,2048,0.030692267417907714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,1536,0.024177066485087075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,16384,0.10353706677754719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,51200,0.7461503982543946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,768,0.015530666708946228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,16384,0.2440330664316813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,12288,0.078111998240153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,512,0.012176000078519185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,10240,0.06579093138376871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,12288,0.18802666664123535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,1024,256,0.0100000003973643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,8192,0.05408746798833212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,10240,0.15803519884745282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,8192,0.12952533562978108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,7168,0.04708906809488932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,7168,0.11532479921976727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,6144,0.04112853209177653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,6144,0.10001813570658366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,12288,0.15101440747578937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,5120,0.03529173135757446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,65536,0.7835039774576823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,16384,0.20280000368754067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,4096,0.02635413408279419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,6144,0.07810666561126708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,4096,0.06677760283152262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,3584,0.023577600717544556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,10240,0.1284437338511149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,3584,0.05818453232447306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,3072,0.020856533447901407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,3072,0.049217065175374344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,7168,0.09037973086039225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,2560,0.017940266927083334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,2560,0.041289599736531575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,5120,0.08464213212331137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,51200,0.6089877446492513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,2048,0.014758400122324624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,2048,0.034886399904886886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,8192,0.1027616024017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,1536,0.012156800429026286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,1536,0.029436800877253217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,1024,0.009335466225941976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,1024,0.02400853236516317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,3072,0.04052053292592366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,768,0.00798933357000351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,768,0.022936532894770302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,1536,0.021817600727081297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,512,0.0067669332027435304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,512,0.020502400398254395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,4096,0.054023468494415285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,256,0.005418666700522105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,256,0.018355200688044228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,5120,0.06539626518885294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,128,0.004744533201058706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,2560,0.03372160196304321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,128,0.016622933745384216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,64,0.004306133091449738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,64,0.016504533092180886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,1024,0.01658986707528432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,768,32,0.004496000210444133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,768,32,0.016731733083724977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,3584,0.04768213431040446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,2048,0.02765333255132039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,65536,0.2827264149983724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,51200,0.2194730599721273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,65536,0.8797887802124024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,51200,0.6957301457722982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,16384,0.08202453454335532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,16384,0.23125866254170738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,12288,0.061869867642720545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,12288,0.17849706013997396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,65536,0.5373760223388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,51200,0.43149334589640304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,16384,0.14578347206115722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,10240,0.0552618662516276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,128,0.007869866490364075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,256,0.009367466966311137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,8192,0.043295999368031815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,10240,0.15315200487772623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,8192,0.07732053597768149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,7168,0.038726401329040525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,8192,0.12178880373636883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,6144,0.033930667241414386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,7168,0.10900479952494305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,12288,0.11350826422373454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,6144,0.09572052955627441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,5120,0.02866986592610677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,768,0.014223999778429666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,768,512,0.011290666460990906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,5120,0.04914559920628865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,5120,0.07982827027638753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,4096,0.01874879995981852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,3584,0.017965867122014364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,4096,0.06292906602223715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,3584,0.05324159860610962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,3072,0.01713493267695109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,3072,0.04514133135477702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,2560,0.013946666320164999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,2560,0.03778666655222575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,2048,0.012081066767374676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,2048,0.031301333506902056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,2048,0.02258239984512329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,1536,0.009586133559544881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,10240,0.09505173365275064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,1536,0.026922667026519777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,1024,0.007257600128650665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,1024,0.022825600703557332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,7168,0.06699626445770264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,768,0.006426666676998138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,768,0.020645334323247274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,6144,0.05897813240687052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,768,0.011938132842381795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,512,0.005365333457787832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,512,0.019350399573644005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,256,0.004605866471926371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,4096,0.04058880011240641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,256,0.017668267091115318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,128,0.004073599974314371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,256,0.008193066716194153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,3584,0.03597226540247599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,128,0.01624853312969208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,64,0.0037610667447249093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,512,32,0.0039658665657043455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,64,0.016017066438992818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,3072,0.03149440089861552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,512,32,0.015986133615175882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,65536,0.16709334055582684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,2560,0.02704319953918457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,51200,0.13372799555460613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,65536,0.8247562408447265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,51200,0.651364262898763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,16384,0.05198826789855957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,16384,0.13773973782857257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,16384,0.22148693402608238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,12288,0.042098132769266765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,128,0.006798933446407318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,12288,0.16998292605082194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,10240,0.0368938684463501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,10240,0.1438111941019694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,8192,0.028061866760253906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,8192,0.11666773160298664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,8192,0.07197973728179932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,7168,0.025997867186864216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,7168,0.1035690704981486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,1536,0.017963733275731406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,6144,0.022056533892949422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,1024,0.014005333185195923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,6144,0.09052159786224365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,5120,0.018776534001032512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,5120,0.07508052984873453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,512,512,0.009750399986902874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,5120,0.04728320042292277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,4096,0.013987200458844504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,4096,0.058050131797790526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,4096,0.03885973294576009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,3584,0.013010133306185404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,3584,0.04935893217722575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,3072,0.012005333105723064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,3072,0.04113279978434245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,65536,0.5246752103169758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,2560,0.009950932860374451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,2560,0.033318400382995605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,51200,0.41517759958903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,2048,0.008707200487454731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,2560,0.02453226645787557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,2048,0.031146667400995892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,1536,0.007134933272997539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,1536,0.025307732820510864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,12288,0.10464213689168293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,1024,0.005827199916044871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,10240,0.08862613042195638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,7168,0.06428800026575723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,6144,0.05533440113067627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,768,0.004999466737111409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,768,0.01978879968325297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,1024,0.02204373280207316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,512,0.004682666560014089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,3584,0.034385065237681076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,512,0.018488534291585288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,256,0.0037802666425704955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,256,0.016622933745384216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,3072,0.029393066962560017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,128,0.0034933333595593774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,128,0.015749333302179973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,2048,0.020617600282033285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,64,0.0033013333876927694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,1536,0.01634773313999176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,64,0.015533866484959922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,256,32,0.0034442665676275887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,256,32,0.01595200002193451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,65536,0.15235946973164877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,1024,0.01258133351802826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,65536,0.4389301300048828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,51200,0.12099839846293133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,65536,0.8135744094848633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,16384,0.04564266602198283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,51200,0.3479210535685221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,16384,0.2146026611328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,51200,0.6403263727823894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,12288,0.037196799119313555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,768,0.01065066655476888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,12288,0.16484586397806805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,10240,0.03211413423220317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,8192,0.02262079914410909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,10240,0.13933973312377929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,8192,0.06069973309834799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,7168,0.020198400815327963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,8192,0.11354026794433594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,512,0.008596266309420269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,7168,0.054041600227355956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,6144,0.014383999506632486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,7168,0.10055253505706788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,5120,0.011534933249155681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,6144,0.08660159905751547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,6144,0.04690239826838176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,4096,0.009648000200589497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,256,0.007207466661930085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,5120,0.0715936024983724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,3584,0.008761599659919739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,256,128,0.0059456000725428265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,4096,0.05450026591618856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,3072,0.008106666803359985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,3584,0.028406399488449096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,3584,0.04579306840896606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,2560,0.009034666419029235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,3072,0.02451840043067932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,2560,0.03181866606076558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,2048,0.007529599964618683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,2048,0.02837439974149068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,2048,0.017873066663742065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,1536,0.006626133124033611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,1536,0.0256117324034373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,10240,0.07586987018585205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,1024,0.004993066688378652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,1024,0.022537599007288613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,4096,0.03224959969520569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,768,0.004624000191688538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,768,0.01960853338241577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,3072,0.038652801513671876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,512,0.004014933357636133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,768,0.010018133123715718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,512,0.008408533533414204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,512,0.01839573383331299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,256,0.003483733286460241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,256,0.016645333170890807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,256,0.007216000060240428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,128,0.003235200047492981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,128,0.015715199708938598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,64,0.0031136001149813337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,16384,0.11732587019602458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,64,0.014999467134475707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,128,32,0.0033514666060606635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,128,32,0.01536853313446045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,65536,0.14688746134440106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,12288,0.08976639906565348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,51200,0.11665066878000896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,16384,0.04174400170644124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,16384,0.2134549299875895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,12288,0.03270399967829386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,51200,0.6360597610473633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,65536,0.8080448150634766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,5120,0.03995413382848104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,10240,0.028553599119186403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,12288,0.16321919759114584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,8192,0.02376000086466471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,7168,0.021410133441289267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,10240,0.13781867027282715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,6144,0.012562132875124612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,8192,0.11251200040181478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,5120,0.010405332843462626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,4096,0.008618666728337606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,6144,0.08606293201446533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,5120,0.07051200071970622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,3584,0.00804373323917389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,4096,0.05370239814122518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,2560,0.021333332856496176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,3072,0.007469866673151653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,3584,0.04509653250376384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,2560,0.007055999835332234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,2048,0.006235733131567637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,3072,0.037573333581288657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,2560,0.03166399995485942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,1536,0.005350400010744731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,2048,0.0284768005212148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,1024,0.004534400006135305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,1536,0.024934399127960204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,768,0.004092800120512644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,512,0.003756800045569738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,1024,0.021626667181650797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,768,0.019113600254058838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,256,0.0033621333539485933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,1536,0.01482133368651072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,128,0.003127466638882955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,512,0.01807360053062439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,256,0.01641279955705007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,64,0.002979200085004171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,64,32,0.0029290666182835894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,128,0.01575573285420736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,64,0.015253333250681558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,32,0.014948266744613647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,65536,0.14784746170043944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,51200,0.11728533109029134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,1024,0.011462400356928509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,16384,0.04151360193888347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,12288,0.03264853358268738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,10240,0.02779200077056885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,16384,0.2131338596343994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,12288,0.16327039400736493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,8192,0.02373653252919515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,10240,0.1381440003712972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,7168,0.020833067099253335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,6144,0.011970133582750956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,8192,0.11220373312632244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,5120,0.010494933525721232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,51200,0.6318122863769531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,7168,0.0987178643544515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,64,7168,0.09976853529612223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,4096,0.009070932865142822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,6144,0.08533439636230469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,3584,0.007726933558781941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,5120,0.07033173243204752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,3072,0.007064533233642578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,2560,0.0062837332487106325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,3584,0.045774932702382407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,3072,0.037511467933654785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,2048,0.005502933263778686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,1536,0.004913066824277243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,2048,0.02820693254470825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,1024,0.004189866781234741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,1536,0.02535039981206258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,768,0.00386559988061587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,1024,0.021465599536895752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,512,0.003587199995915095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,768,0.019318399826685588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,256,0.003236266722281774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,512,0.017090133825937905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3072,128,128,0.006159999966621399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,256,0.01606186628341675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,64,0.0027872001131375628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,128,0.01527466674645742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,32,0.0027882667879263563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,64,0.014910933375358582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,32,0.015290666619936624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,65536,0.8095658620198568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,2560,0.03248320023218791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3072,32,128,0.0029685333371162414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3072,32,4096,0.05299413204193115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,16384,4.368317667643229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,12288,3.338063049316406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,12288,6.0746912638346355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,16384,8.043954976399739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,12288,4.421971130371094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,16384,6.0403564453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,10240,2.8058517456054686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,10240,5.002418009440104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,8192,2.172637939453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,10240,3.9711743672688806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,8192,2.8646176656087237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,51200,14.244931030273438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,8192,4.349666341145833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,7168,3.600585683186849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,7168,1.9209823608398438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,7168,2.9815210978190105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,5120,1.3835882822672525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,6144,1.7978410085042316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,6144,3.532726287841797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,6144,2.44905268351237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,5120,3.053209686279297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,4096,1.1226336161295571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,4096,1.4827147165934245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,3584,0.9574752171834311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,4096,2.5032310485839844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,3584,1.8474815368652344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,3584,1.325202178955078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,3072,0.8624095916748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,3072,1.0996981302897135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,3072,1.5883008321126302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,2560,1.2812095642089845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,2560,0.7226165135701497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,5120,1.8559477488199871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,2048,0.6071381251017253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,2560,0.9399295806884765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,2048,1.058570671081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,2048,0.8081621170043946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,1536,0.44948479334513347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,1024,0.3300554593404134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,1024,0.5627967834472656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,1536,0.812281608581543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,768,0.40431467692057294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,1024,0.4809386571248372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,768,0.2796693483988444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,512,0.19436906178792318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,512,0.30914878845214844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,256,0.18037439982096354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,512,0.32815891901652017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,256,0.17298879623413085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,128,0.16808959643046062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,256,0.30950826009114585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,128,0.2626314640045166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,64,0.15508480072021485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,64,0.1650826613108317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,32,0.1514431953430176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,65536,32,0.16045653025309245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,128,0.15870505968729656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,51200,26.49752197265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,1536,0.5978378931681315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,65536,51200,26.044767252604167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,65536,768,0.40957333246866867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,16384,3.7075520833333337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,16384,6.986885579427083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,16384,4.549638366699218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,51200,11.768002319335938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,12288,2.833817545572917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,12288,4.991316223144532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,65536,15.529265340169271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,10240,2.3995424906412763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,10240,2.944024658203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,10240,4.4467625935872395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,8192,1.8662335713704425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,12288,3.467918904622396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,51200,20.350213623046876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,8192,3.6746421813964845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,8192,2.254017130533854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,51200,22.356620279947915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,6144,1.3537867228190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,7168,1.6130346934000652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,7168,1.9902751922607422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,7168,2.9158315022786456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,6144,2.7265589396158854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,5120,1.2006666819254557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,5120,1.3682613372802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,4096,0.9408437093098959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,5120,2.1566017150878904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,4096,1.6926751454671223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,4096,1.1879104614257812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,3584,0.7850400288899739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,65536,26.038234456380206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,3584,1.0221941630045572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,3584,1.5930997212727864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,3072,0.752244249979655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,3072,0.8626154581705728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,2560,0.5820469538370768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,65536,27.10109049479167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,2560,1.0686570485432942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,2048,0.4962954521179199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,2560,0.7351690928141277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,2048,0.923625628153483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,2048,0.6534922917683919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,1536,0.37617174784342444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,6144,1.69868901570638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,1024,0.2990528106689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,1024,0.4541280110677083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,1536,0.49217065175374347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,1024,0.3888479868570963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,1536,0.6582207997639974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,3072,1.3833141326904297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,512,0.2621215979258219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,768,0.3302271842956543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,512,0.1936800003051758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,768,0.2595797379811605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,256,0.1474570592244466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,256,0.17911252975463868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,512,0.27314879099527994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,128,0.13419413566589355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,64,0.12268799940745037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,128,0.13665599822998048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,51200,32,0.12103892962137859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,256,0.25755626360575357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,128,0.218178129196167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,64,0.1522666613260905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,51200,32,0.13121600151062013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,16384,2.1563957214355467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,51200,768,0.32307093938191733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,51200,3.7440127054850256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,16384,1.2341173807779948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,65536,4.81235097249349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,16384,1.4431957244873046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,12288,1.6691818237304688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,12288,0.8817802429199219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,10240,0.7812234878540039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,51200,6.571813456217448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,12288,1.115768559773763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,10240,1.231317392985026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,65536,7.314964294433594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,8192,0.6156469345092773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,8192,0.7240351994832357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,8192,1.071347173055013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,7168,0.5307903925577799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,65536,8.133835856119791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,7168,0.6178656260172526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,6144,0.48841066360473634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,6144,0.5386986414591471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,51200,5.203648885091146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,6144,0.814081064860026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,10240,0.9565557479858399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,5120,0.4150239944458008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,5120,0.4661151885986328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,5120,0.6150208155314127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,4096,0.3093653361002604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,4096,0.3892565409342448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,4096,0.48792212804158525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,3584,0.2994037310282389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,7168,0.9619893391927083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,3072,0.23769706090291343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,3072,0.3051722526550293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,3072,0.3698965390523275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,3584,0.3546367963155111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,2560,0.220361598332723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,2048,0.2465888023376465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,2048,0.21815892855326333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,2048,0.17480212847391766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,1536,0.13818666140238445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,1024,0.13427732785542806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,1024,0.10589120388031006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,1536,0.1656885306040446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,1024,0.13218453725179036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,768,0.10668373107910156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,512,0.07819093068440755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,768,0.08733759721120199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,256,0.05134079853693644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,512,0.06832959651947021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,512,0.09657279650370279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,128,0.04371626774470012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,256,0.058747732639312746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,256,0.07929920355478923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,64,0.0401962677637736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,128,0.05392426649729411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,128,0.07399253050486246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,3584,0.4265418688456218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,32,0.04037653207778931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,64,0.05381120045979818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,16384,32,0.054866135120391846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,2560,0.331989320119222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,2560,0.23794347445170083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,16384,1536,0.20889172554016114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,16384,768,0.11392107009887695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,51200,3.062902323404948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,16384,1.6255306243896483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,16384,0.9295520146687826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,65536,3.768597412109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,51200,3.467327880859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,12288,0.7222976048787435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,12288,0.7834346771240235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,12288,1.1844170888264975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,51200,5.142149353027344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,10240,0.9557472229003906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,10240,0.5951978683471679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,65536,5.252153523763021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,10240,0.6945695877075195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,8192,0.4934282620747884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,8192,0.7887840270996094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,8192,0.5564725240071614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,7168,0.43936745325724286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,7168,0.6847797393798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,65536,6.098240152994792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,7168,0.5085898717244466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,6144,0.4018858591715495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,6144,0.5657354354858398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,5120,0.33122132619222006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,6144,0.4377269426981608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,5120,0.4636831919352214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,4096,0.2636064052581787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,4096,0.40417919158935545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,3584,0.24524799982706705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,5120,0.3609461466471354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,3584,0.34611733754475915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,3584,0.26225919723510743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,3072,0.28863999048868816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,3072,0.22986666361490884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,2560,0.1771008014678955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,2560,0.19612906773885092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,2560,0.24632320404052735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,2048,0.20501333872477212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,2048,0.14795519510904948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,1536,0.1481887976328532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,2048,0.16666026115417482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,1024,0.10548693339029949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,1024,0.08761066595713297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,1536,0.12215786774953205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,1536,0.13059093157450358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,768,0.08387413024902343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,1024,0.10496533711751302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,768,0.07898666858673095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,768,0.09036800066630045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,512,0.06291733185450235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,256,0.041493332386016844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,512,0.061595733960469565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,512,0.08148480256398519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,256,0.05021760066350302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,128,0.03506879806518555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,256,0.06091733376185099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,128,0.045739734172821046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,128,0.05755626757939657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,64,0.03100373347600301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,12288,32,0.031726932525634764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,64,0.04619413216908773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,16384,1.0784170786539713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,32,0.04537493387858073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,12288,4096,0.3048159917195638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,12288,3072,0.21081066131591797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,16384,1.245081583658854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,51200,2.5600128173828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,16384,0.840117327372233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,51200,2.947496541341146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,16384,0.8782229105631509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,12288,0.6357642491658528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,65536,3.4609621683756506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,12288,0.9465194702148437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,10240,0.5286826769510905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,12288,0.6674079895019531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,65536,4.215443166097005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,51200,4.149783579508464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,10240,0.5850175857543946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,10240,0.8159327824910483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,8192,0.6616896311442058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,8192,0.42220586140950517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,8192,0.4648426691691081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,7168,0.5809408187866211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,7168,0.42334400812784834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,7168,0.3892437299092611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,6144,0.337993590037028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,6144,0.4985482533772786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,6144,0.34930454889933266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,5120,0.4420778592427571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,4096,0.22411093711853028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,5120,0.28680426279703775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,5120,0.30369173685709633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,4096,0.3197866757710775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,3584,0.2874368031819662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,3584,0.21308479309082032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,3584,0.20707947413126626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,3072,0.19660693804423016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,3072,0.1829578717549642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,3072,0.24284906387329103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,2560,0.21910506884256997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,2560,0.16314026514689128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,2560,0.15790079434712728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,2048,0.17346347173055013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,2048,0.1493824005126953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,1536,0.13991573651631672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,1536,0.10693226655324299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,1536,0.10529066721598308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,1024,0.08447253704071045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,1024,0.098089599609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,768,0.07017066478729247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,768,0.06632533470789591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,512,0.0581055998802185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,768,0.07895893255869547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,512,0.05220160086949667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,256,0.03749653498331706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,512,0.06382506688435872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,256,0.04568106730779012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,128,0.030558933814366657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,256,0.04990400075912475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,128,0.04059413274129232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,64,0.026986666520436603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,128,0.05146666765213013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,32,0.028085333108901978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,64,0.041221332550048825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,32,0.04188160101572673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,65536,5.250399780273438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,10240,2048,0.1329792022705078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,10240,1024,0.0961354653040568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,16384,1.0311317443847656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,10240,4096,0.2306879997253418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,51200,2.1205066680908202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,16384,0.7141600290934245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,65536,2.7456586201985678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,65536,3.231344095865885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,51200,3.3401397705078124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,51200,2.458934275309245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,12288,0.5237194697062175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,12288,0.778553581237793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,12288,0.5733749389648437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,10240,0.45449066162109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,8192,0.36620585123697913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,10240,0.6684384028116862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,10240,0.4992181460062663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,8192,0.5166336059570312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,65536,4.132860819498697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,8192,0.37259413401285807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,7168,0.336136531829834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,7168,0.4415178616841634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,6144,0.393827215830485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,7168,0.36466986338297525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,6144,0.2960629463195801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,6144,0.3115455945332845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,5120,0.24037013053894044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,5120,0.3222389221191406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,4096,0.26422719955444335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,5120,0.25636587142944334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,4096,0.19220159848531088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,4096,0.2147530714670817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,3584,0.1719818592071533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,3072,0.14788907368977863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,3072,0.19501652717590331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,3584,0.19092267354329426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,2560,0.16755520502726237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,2560,0.1272426684697469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,3072,0.16532373428344727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,1536,0.1063424030939738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,2048,0.10828800201416015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,2048,0.11792319615681965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,1024,0.07284479935963949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,1536,0.08755733172098795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,1536,0.0936522642771403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,1024,0.06686399777730306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,768,0.057486931482950844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,1024,0.07345279852549234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,512,0.04356480042139689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,768,0.05668160120646158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,768,0.06476159890492758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,512,0.04758933385213216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,256,0.029255467653274535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,512,0.05333226521809896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,128,0.025681066513061523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,256,0.03890346686045329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,256,0.044737064838409425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,128,0.03583893378575643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,64,0.02186773419380188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,128,0.041731198628743485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,32,0.022776534159978233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,64,0.03607573509216309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,8192,32,0.036305065949757895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,3584,0.22756800651550294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,8192,2048,0.13827306429545086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,2560,0.14760212898254393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,8192,16384,0.7304906845092773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,16384,0.8925898869832357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,16384,0.6066794713338216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,16384,0.6802378972371419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,65536,2.4280298868815104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,51200,1.9524959564208983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,51200,2.080940755208333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,65536,2.691201019287109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,12288,0.6388202667236328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,12288,0.5055893262227376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,51200,2.8482208251953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,10240,0.4330154736836751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,12288,0.48879146575927734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,10240,0.5580650965372722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,8192,0.32933546702067057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,10240,0.4133920033772786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,8192,0.4711114565531413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,7168,0.2870346705118815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,65536,3.573627726236979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,7168,0.3898357391357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,7168,0.3127690633138021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,8192,0.3287626584370931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,6144,0.3194058736165365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,6144,0.26103787422180175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,6144,0.27334826787312827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,5120,0.26771628061930336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,4096,0.2325823942820231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,4096,0.17834240595499676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,5120,0.21265066464742027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,5120,0.2188096046447754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,3584,0.1973098595937093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,3584,0.1642858664194743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,3584,0.1595413366953532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,3072,0.17938133875528972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,3072,0.13362453778584799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,2560,0.11691093444824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,2048,0.11782399813334148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,2048,0.10353493690490723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,1536,0.0909279982248942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,2048,0.10064319769541423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,1536,0.08219733238220214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,1024,0.06461653312047323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,1536,0.08527680238087973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,1024,0.06221333344777426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,768,0.051620264848073326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,768,0.05386026700337728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,768,0.057860267162322995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,512,0.039828264713287355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,512,0.043350398540496826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,512,0.04766186475753784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,256,0.025962666670481367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,256,0.03691413402557373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,256,0.03828159968058269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,128,0.02327573299407959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,128,0.03336853186289469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,128,0.03657173315684001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,64,0.01939093271891276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,32,0.02093120018641154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,64,0.03436160087585449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,7168,32,0.034873600800832114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,4096,0.1895125389099121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,7168,2560,0.1385973294576009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,2560,0.129094401995341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,3072,0.1345429261525472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,51200,1.7181120554606122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,65536,2.2172757466634114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,7168,1024,0.06381866534550985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,51200,2.510304005940755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,16384,0.7246293385823568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,16384,0.5571317036946615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,65536,3.2347732543945313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,16384,0.56659730275472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,12288,0.41327571868896484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,12288,0.5773546854654948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,12288,0.45062719980875654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,10240,0.46641171773274737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,10240,0.37454506556193035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,10240,0.3830975850423177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,8192,0.3621983846028646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,8192,0.30987412134806314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,8192,0.3043903986612956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,7168,0.35074027379353845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,7168,0.2825173377990723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,6144,0.22597546577453614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,6144,0.2748682657877604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,6144,0.23203733762105308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,5120,0.2546154657999674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,5120,0.2062922636667887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,4096,0.1855829397837321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,5120,0.19002559979756672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,4096,0.1537109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,3584,0.1651957352956136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,3584,0.14332799911499022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,3072,0.1433685302734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,3072,0.12217173576354981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,3072,0.12806826432545979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,2560,0.12100906372070312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,2560,0.10633066495259602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,2048,0.09885866641998291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,2560,0.11020373503367106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,1536,0.0788149356842041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,2048,0.0927285353342692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,2048,0.09101013342539468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,1024,0.055386666456858316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,1536,0.0748586654663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,1536,0.0764522631963094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,768,0.04647146860758464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,1024,0.05662293434143066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,1024,0.06112213134765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,512,0.034017066160837814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,768,0.0499509334564209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,768,0.050188799699147545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,7168,0.2624010721842448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,256,0.023885866006215416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,512,0.04310719966888428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,512,0.04243413209915161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,256,0.03514773448308309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,256,0.03429226477940877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,128,0.01906986633936564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,128,0.03128746747970581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,64,0.016720000902811685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,128,0.032893866300582886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,64,0.031464533011118574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,6144,32,0.017245866854985557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,6144,32,0.030852266152699787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,65536,2.4484181722005207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,51200,1.8705652872721354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,51200,1.5173824310302735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,65536,1.930941899617513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,3584,0.1366048018137614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,65536,2.0045706431070967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,6144,4096,0.15499733289082845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,51200,2.167430369059245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,12288,0.3730549176534017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,16384,0.6490933100382488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,65536,2.7103734334309895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,16384,0.5044469197591146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,12288,0.5206720034281414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,10240,0.38079252243041994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,8192,0.30668694178263345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,10240,0.34337921142578126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,8192,0.2850559870402018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,8192,0.25846293767293294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,7168,0.2828768094380697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,7168,0.242851193745931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,6144,0.2516981283823649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,6144,0.21714879671732584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,5120,0.2009866714477539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,5120,0.1760533332824707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,5120,0.1606709321339925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,4096,0.15676479339599608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,4096,0.13691840171813965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,4096,0.13133546511332195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,3584,0.13873066902160644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,3584,0.1207487980524699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,16384,0.45786558787028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,12288,0.35356480280558267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,3072,0.12840533256530762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,3072,0.10703466733296711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,51200,1.488430913289388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,2560,0.10340373516082764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,2560,0.11252053578694661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,2560,0.0870965321858724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,10240,0.29466772079467773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,2048,0.10259626706441242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,2048,0.08012373447418213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,2048,0.072762664159139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,1536,0.07330880165100098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,1024,0.053895465532938634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,1536,0.0597109317779541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,1536,0.06576639811197917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,7168,0.2125472068786621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,768,0.038791465759277347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,1024,0.05442453225453695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,1024,0.047353601455688475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,768,0.04437973499298096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,512,0.0318229337533315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,768,0.04145280122756958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,512,0.034777601559956864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,512,0.03755199909210205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,256,0.020935465892155967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,128,0.016218666235605875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,256,0.028564266363779706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,128,0.028922667105992634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,128,0.026497066020965576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,64,0.01448853313922882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,5120,32,0.015225600202878317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,64,0.028545065720876055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,32,0.02924586733182271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,6144,0.18498454093933106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,3072,0.10784853299458821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,5120,256,0.03445546627044678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,51200,1.319707743326823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,51200,1.6651093800862629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,51200,1.292022450764974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,65536,1.6866058349609374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,16384,0.47850453058878584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,65536,2.2108299255371096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,16384,0.44040854771931964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,12288,0.3348544120788574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,12288,0.40752534866333007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,12288,0.31760212580362956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,10240,0.2783498764038086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,10240,0.31352640787760416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,10240,0.2582250595092773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,8192,0.26729278564453124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,8192,0.24308692614237465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,7168,0.2199168046315511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,7168,0.20346666971842448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,7168,0.2026133378346761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,6144,0.19781972567240397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,6144,0.16285227139790853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,5120,0.16281174023946127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,6144,0.18401494026184081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,5120,3584,0.11496000289916992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,4096,0.13224960168202718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,5120,0.13707413673400878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,4096,0.12309866746266682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,3584,0.11538879871368408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,3584,0.10269013245900471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,3584,0.10710293451944988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,3072,0.10111040274302166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,3072,0.0871392011642456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,3072,0.09484373728434245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,2560,0.08743680318196614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,2048,0.07195306619008382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,2560,0.08376320203145346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,16384,0.4445055961608887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,2048,0.07540799776713053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,1536,0.05216853221257528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,1536,0.062165331840515134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,1536,0.05241386493047079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,1024,0.03905066649119059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,1024,0.045187199115753175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,1024,0.04331839879353841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,768,0.030747733513514203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,768,0.04012266794840495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,768,0.03594026565551758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,512,0.02451200087865194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,512,0.03095573385556539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,256,0.01738026738166809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,5120,0.15755947430928546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,256,0.029202133417129517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,128,0.01453439990679423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,128,0.026997333765029906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,65536,1.733026123046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,128,0.0291701336701711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,64,0.01458346645037333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,64,0.027130667368570966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,4096,32,0.013636266191800436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,32,0.027062400182088213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,4096,512,0.03257493376731872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,8192,0.20540800094604492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,4096,0.11166720390319824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,2560,0.07585706710815429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,2048,0.06375466585159302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,65536,1.9382432301839192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,65536,1.5112757364908853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,65536,1.4590571085611979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,51200,1.3813194274902343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,16384,0.4743061383565267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,4096,256,0.02502506573994954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,16384,0.3969802538553874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,51200,1.1846368153889975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,12288,0.3517066637674967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,12288,0.265010134379069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,10240,0.2575263977050781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,12288,0.3271327972412109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,8192,0.20735360781351725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,10240,0.23882026672363282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,10240,0.2540138721466064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,8192,0.1903125286102295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,8192,0.20994879404703776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,7168,0.1905248006184896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,6144,0.15956586201985676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,7168,0.16106665929158528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,6144,0.14395947456359864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,6144,0.16345280011494953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,7168,0.19461545944213868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,5120,0.13849600156148273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,5120,0.1349237283070882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,5120,0.11892480055491131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,4096,0.11041920185089112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,3584,0.09974506696065268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,3584,0.09930346806844076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,3072,0.0837930679321289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,3072,0.09090987046559652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,2560,0.07086293697357178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,3072,0.08038613001505533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,2560,0.07986453374226889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,2048,0.058822401364644374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,2048,0.05960640112559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,2048,0.06576106548309327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,1536,0.04503999948501587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,1536,0.05249919891357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,1536,0.047653333346048994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,1024,0.03297280073165894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,1024,0.04012480179468791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,1024,0.03789973258972168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,768,0.026281599203745527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,768,0.03415253162384033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,768,0.03277759949366252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,512,0.02016213337580363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,512,0.030202666918436687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,512,0.02678826649983724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,256,0.013821867108345032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,256,0.026539733012517292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,256,0.02226346731185913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,128,0.011762133240699768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,128,0.024759467442830405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,64,0.011354666948318482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,64,0.024626133839289348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,4096,0.1067733367284139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3584,32,0.01169706682364146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3584,32,0.024858667453130086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,2560,0.06813653310139975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,16384,0.3512682596842448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,51200,1.1314047495524089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,128,0.020334933201471964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,65536,1.6203498840332031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,65536,1.3982048034667969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,4096,0.09925333658854166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3584,3584,0.08652160167694092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,16384,0.361023998260498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,16384,0.4158485412597656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,51200,1.1633557637532552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,12288,0.2652234713236491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,51200,1.1248181660970054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,12288,0.2753408114115397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,10240,0.2623114744822184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,10240,0.23550400733947754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,8192,0.17948266665140789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,8192,0.2158005396525065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,7168,0.1580512046813965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,7168,0.1803541342417399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,6144,0.14745173454284669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,6144,0.14924373626708984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,65536,1.808528010050456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,12288,0.3179274559020996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,10240,0.26279786427815754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,5120,0.13080000082651774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,5120,0.11960533459981282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,51200,1.3170730590820312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,4096,0.0962282657623291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,8192,0.20393919944763184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,3584,0.08531413078308106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,4096,0.09878826936086019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,3584,0.09358932971954345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,7168,0.1789994716644287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,3072,0.07329813639322916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,3072,0.08001279830932617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,2560,0.06005333264668783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,2560,0.06905813217163086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,6144,0.15753386815388998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,2048,0.04991893370946248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,16384,0.40366398493448896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,2048,0.058721065521240234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,1536,0.03882773319880168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,1536,0.047416532039642335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,5120,0.12950719992319742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,1024,0.027654399474461872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,1024,0.037062398592631024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,3584,0.0915712038675944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,768,0.022818134228388468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,768,0.03179306586583455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,3072,0.07842880090077718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,512,0.017760000626246133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,512,0.020784000555674233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,512,0.02775999903678894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,256,0.012072533369064331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,256,0.022846933205922446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,128,0.00953493316968282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,2560,0.06873599688212076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,128,0.0219541331132253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,128,0.014032000303268432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,64,0.007996800045172375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,3072,32,0.008473599950472515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,64,0.02183253367741903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,2048,0.05456533432006836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,3072,32,0.02136639952659607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,1536,0.04096320072809855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,4096,0.10715946356455486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,51200,0.9670719782511392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,65536,1.2725685119628907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,51200,0.9927487691243491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,65536,1.4545771280924478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,16384,0.29744958877563477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,16384,0.328983465830485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,12288,0.22793706258138022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,16384,0.3871349334716797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,12288,0.25001494089762366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,10240,0.19204586346944172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,1024,0.034406399726867674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,10240,0.21028587023417153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,8192,0.149618132909139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,768,0.026027733087539674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,8192,0.1709226608276367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,65536,1.3012330373128256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,7168,0.13292053540547688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,8192,0.17488853136698407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,7168,0.1502687931060791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,6144,0.11485226949055989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,3072,256,0.016523733735084534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,6144,0.12990612983703614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,5120,0.09674879709879557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,5120,0.10954773426055908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,4096,0.07851413091023764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,4096,0.09082240263621012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,4096,0.09662613073984781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,3584,0.06762773195902506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,3584,0.08338987032572429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,3072,0.05981013377507528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,3072,0.07220373153686524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,3072,0.06954452991485596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,2560,0.050508801142374674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,2560,0.06220373312632242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,5120,0.11434133052825927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,2048,0.04146666526794433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,2048,0.0515882651011149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,12288,0.26371307373046876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,1536,0.031768532594045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,1536,0.04138026634852092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,1536,0.03837546507517497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,10240,0.22232960065205892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,1024,0.02294506629308065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,1024,0.03268373409907023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,1024,0.030065067609151203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,768,0.019002666076024376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,768,0.02847040096918742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,512,0.014528000354766845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,512,0.024447999397913613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,6144,0.13256746927897137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,256,0.010095999638239542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,256,0.02202346722284953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,256,0.016247466206550598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,128,0.008028799792130788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,128,0.02116159995396932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,3584,0.08228906790415445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,64,0.007673599819342296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,64,0.02071466644605001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2560,32,0.008224000036716462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2560,32,0.021065600713094077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,51200,1.1325108846028646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,2560,0.06267306804656983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,7168,0.16100266774495442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,65536,1.0546538670857748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,2048,0.04957120021184285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,65536,0.9512917200724283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,65536,1.2382165273030599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,16384,0.245796267191569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,51200,0.8054112116495767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,51200,0.7634186426798503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,16384,0.3039029439290365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,768,0.02337813377380371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,16384,0.28857386906941734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,12288,0.18487359682718912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,12288,0.23387519518534342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,10240,0.1735040028889974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,12288,0.1785077253977458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,512,0.018887466192245482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,8192,0.13090346654256185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,10240,0.15261972745259603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,10240,0.20623359680175782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,8192,0.1587392012278239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,7168,0.11572799682617188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,7168,0.14044267336527508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,6144,0.10085333188374837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2560,128,0.013529599706331889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,6144,0.09452693462371826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,5120,0.08040320078531901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,6144,0.13241600195566813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,5120,0.10119679768880208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,5120,0.08056639830271403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,4096,0.08673066298166911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,4096,0.06562026739120483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,3584,0.05764906803766886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,3584,0.07747519810994466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,3072,0.04957546790440877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,3072,0.05241706768671671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,3072,0.06767253081003824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,2560,0.04109866619110107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,2560,0.056790399551391604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,51200,0.9520800272623697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,2048,0.032841600974400836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,2048,0.04036266803741455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,2048,0.04500159819920858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,1536,0.026415999730428057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,1536,0.03667200009028117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,1024,0.01830293337504069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,1024,0.028732800483703615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,1024,0.024987733364105223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,768,0.014852266510327658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,768,0.02632533311843872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,768,0.021358933051427206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,512,0.011065600315729777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,512,0.023050665855407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,4096,0.06517653465270996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,512,0.018088533480962118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,256,0.015339733163515726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,128,0.007162666817506154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,128,0.01923840045928955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,128,0.013591466347376504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,64,0.006437333424886067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,64,0.021824000279108684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,32,0.009142399827639262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,32,0.021997867027918498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,8192,0.12299199899037679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,2048,256,0.008327466746171314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,7168,0.10891199906667073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,2048,256,0.020551466941833497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,3584,0.05887146790822347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,51200,0.5686560312906901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,2560,0.04566293160120646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,16384,0.19094400405883788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,65536,1.0234901428222656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,51200,0.8132074356079102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,2048,1536,0.031787733236948654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,12288,0.13422080675760906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,16384,0.26966400146484376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,12288,0.20800959269205727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,10240,0.11280852953592937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,8192,0.091483736038208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,10240,0.17404054005940756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,65536,0.7051413218180339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,7168,0.07960320313771566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,8192,0.14500160217285157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,7168,0.12629120349884032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,6144,0.06886293093363444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,6144,0.11012372970581055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,65536,0.8348416010538736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,5120,0.05816533168156942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,16384,0.21679360071818032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,5120,0.09325760205586751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,4096,0.04734293222427368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,12288,0.16314773559570311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,4096,0.07571840286254883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,3584,0.04195626576741536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,10240,0.14008746147155762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,3584,0.06694506804148356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,3584,0.05448640187581381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,3072,0.03657919963200887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,8192,0.11445653438568115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,3072,0.0590399980545044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,2560,0.031045333544413252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,3072,0.04763946533203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,7168,0.10023252964019776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,2560,0.0493343989054362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,2048,0.02574719985326131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,2048,0.04111573298772176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,1536,0.021215999126434328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,1536,0.03396586577097575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,6144,0.08519039948781332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,1024,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,1024,0.02767573396364848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,51200,0.6686634699503581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,768,0.012573867042859396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,768,0.024114133914311726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,5120,0.07320640087127686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,512,0.010025599598884582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,4096,0.059455998738606776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,512,0.022077866395314536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,256,0.007574399809042613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,256,0.019474132855733236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,2560,0.041145598888397215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,128,0.0063178668419520065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,2048,0.03380693197250366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,128,0.018412800629933675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,64,0.005579733351866404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,64,0.01826559901237488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1536,32,0.006073600053787232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1536,32,0.01815999945004781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,1024,0.021976532538731892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,65536,0.482967472076416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,768,0.017704532543818156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,51200,0.4034079869588216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,65536,0.9274005254109701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,16384,0.12378346920013428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,51200,0.7252362569173176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,1536,0.027149866024653118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,16384,0.23861546516418458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,12288,0.09471680323282877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,12288,0.18313387235005696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,10240,0.07941226959228516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,10240,0.15472319920857747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,512,0.01518933375676473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,8192,0.06255893309911092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,256,0.012616533041000366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,8192,0.127237335840861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,7168,0.054980266094207766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,8192,0.10829866727193196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,7168,0.1130954662958781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,6144,0.0473034660021464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1536,128,0.010769066214561463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,6144,0.08056426843007405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,6144,0.09834880034128825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,5120,0.04086826642354329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,5120,0.08241173426310221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,4096,0.03233493367830913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,4096,0.06604479948679606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,65536,0.8202367782592773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,3584,0.028701867659886676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,3584,0.05904959837595621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,16384,0.20872960090637208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,51200,0.6341642379760742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,3072,0.024961066246032716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,12288,0.15501866340637208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,3072,0.049652266502380374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,2560,0.021993599335352578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,2560,0.04165866772333781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,2048,0.018105600277582803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,2048,0.035634132226308186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,10240,0.13226453463236493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,1536,0.014625066518783569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,1536,0.02995413343111674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,7168,0.09505066871643067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,1024,0.010857599973678588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,1024,0.02472533384958903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,1024,0.017876267433166504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,768,0.009340799848238627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,768,0.022206934293111165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,768,0.014793599645296732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,5120,0.06746773719787598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,512,0.007624533275763195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,512,0.020362667242685952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,256,0.006123733520507812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,256,0.018489599227905273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,3584,0.04881173372268677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,128,0.005276800195376078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,3072,0.042138667901357015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,128,0.01775040030479431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,64,0.004752000172932943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,64,0.017310933272043864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,1024,32,0.005125333368778229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,4096,0.05688639879226685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,1024,32,0.01780160069465637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,2048,0.029278934001922607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,65536,0.36447572708129883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,1536,0.02298240065574646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,51200,0.29028374354044595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,65536,0.8600394566853842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,16384,0.09405013720194498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,51200,0.6738645553588867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,2560,0.03594880104064942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,16384,0.2230442682902018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,12288,0.07252906958262126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,12288,0.17190507253011067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,512,0.011994666854540507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,10240,0.05988479852676391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,10240,0.14581972757975262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,256,0.009992532928784688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,8192,0.048522667090098066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,8192,0.12013013362884521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,1024,128,0.008409600456555684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,7168,0.04229546785354614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,7168,0.10714560349782307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,65536,0.7501301447550456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,6144,0.037013332049051925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,6144,0.09335467020670572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,6144,0.0750485340754191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,5120,0.03128106594085693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,5120,0.07844693660736084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,16384,0.19261013666788737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,4096,0.025388799111048382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,4096,0.06212160189946493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,12288,0.1467360019683838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,3584,0.022731733322143555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,3584,0.0536789337793986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,3072,0.02065066695213318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,3072,0.045238399505615236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,10240,0.1232682704925537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,2560,0.016960000991821288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,2560,0.03834240039189656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,8192,0.0992415984471639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,2048,0.014131200313568116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,51200,0.5874314626057943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,2048,0.03322666684786479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,1536,0.011462400356928509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,1536,0.027646932999293012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,7168,0.08762666384379068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,1024,0.009004799524943034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,1024,0.0240064005057017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,5120,0.0635968009630839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,768,0.007682133217652638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,768,0.021625600258509316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,4096,0.052293332417805996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,3584,0.045422931512196854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,512,0.01946880022684733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,3072,0.03883413473765056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,256,0.005297066768010458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,256,0.017781333128611247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,2560,0.03311040004094441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,128,0.0045962666471799215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,128,0.0166485329469045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,2048,0.02685760060946147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,64,0.004227200150489807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,64,0.01665600041548411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,1536,0.021025067567825316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,768,32,0.016718933979670204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,512,0.006490666667620342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,65536,0.25630079905192055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,1024,0.016736000776290894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,51200,0.20050986607869467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,65536,0.5243904113769531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,65536,0.8114378611246744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,16384,0.06762986977895101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,51200,0.6375584284464518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,51200,0.4207327842712402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,768,32,0.004376533130804697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,12288,0.051541332403818765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,16384,0.21100373268127443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,12288,0.16298346519470214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,10240,0.04328320026397705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,10240,0.13876800537109374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,768,0.013707733154296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,8192,0.03467626571655273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,8192,0.11403946876525879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,512,0.01104853351910909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,7168,0.031115732590357464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,7168,0.06617386738459269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,7168,0.10096426804860432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,6144,0.027220267057418823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,256,0.009251200159390767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,6144,0.057385599613189696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,6144,0.08749547004699706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,5120,0.0220085342725118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,5120,0.07270932992299398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,5120,0.04906133413314819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,768,128,0.007854933540026348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,4096,0.04145386616388957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,4096,0.05662186543146769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,3584,0.01655466655890147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,3072,0.014391466975212097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,3584,0.03513706525166829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,3584,0.048485334714253744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,2560,0.012899200121561686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,3072,0.041124268372853594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,2560,0.03499946594238281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,2048,0.010945066809654236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,2048,0.030220800638198854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,4096,0.018195199966430663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,1536,0.00907306671142578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,1536,0.02592639923095703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,1536,0.017870932817459106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,1024,0.007225599884986877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,2560,0.02648959954579671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,1024,0.022369066874186196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,16384,0.14580693244934081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,768,0.006486399968465169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,12288,0.11080106894175212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,768,0.020283732811609903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,512,0.0055637334783871974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,512,0.018662399053573607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,10240,0.09240533510843912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,512,0.009482666850090027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,256,0.0046015997727712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,256,0.017697066068649292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,128,0.004033066580692927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,8192,0.07486826578776042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,128,0.016833066940307617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,64,0.004058666775623957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,64,0.016522666811943053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,512,32,0.004101333270470301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,512,32,0.016119466225306193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,65536,0.15437332789103192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,3072,0.030667734146118165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,768,0.011382399996121725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,2048,0.02193386753400167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,51200,0.1245194673538208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,65536,0.7596874872843424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,65536,0.5008501370747884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,16384,0.04904746611913045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,16384,0.20215466817220054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,12288,0.034891732533772785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,51200,0.5966474533081054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,12288,0.15611519813537597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,12288,0.09921600023905436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,10240,0.029612799485524494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,10240,0.08442453543345133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,8192,0.02404586672782898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,10240,0.13119786580403645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,8192,0.10796693166097004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,7168,0.021927465995152794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,1024,0.014106667041778565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,7168,0.09495893319447836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,6144,0.01604373355706533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,6144,0.08192959626515707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,5120,0.014087466398874917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,5120,0.06709226767222086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,128,0.006712533533573151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,4096,0.011693867047627766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,512,256,0.008026666442553202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,4096,0.05065493186314901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,3584,0.010314666231473287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,3584,0.04291413227717082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,3584,0.032228267192840575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,3072,0.009556266665458679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,3072,0.036824532349904376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,16384,0.1330015977223714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,2560,0.008550399541854858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,51200,0.39939839045206704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,2560,0.03183786670366923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,2048,0.007481599847475688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,2048,0.028945066531499225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,8192,0.07001492977142335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,1536,0.006393600006898243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,1536,0.024541866779327393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,1536,0.015636266271273295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,1024,0.005407999952634176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,7168,0.061510399977366126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,1024,0.021432532866795858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,768,0.004917333523432413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,6144,0.0525546669960022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,768,0.01982933282852173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,512,0.004377600053946177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,512,0.018066134055455527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,512,0.00848426620165507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,256,0.0038912000755469004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,256,0.016583466529846193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,4096,0.037595733006795244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,128,0.0036586667100588477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,128,0.01606186628341675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,128,0.005975466469923655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,64,0.0033973333736260734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,3072,0.026778666178385417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,256,32,0.0034101332227389016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,64,0.016032000382741295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,256,32,0.015131733814875283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,65536,0.15873707135518392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,5120,0.04501013358434041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,65536,0.4131477355957031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,51200,0.1238474686940511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,65536,0.7467498779296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,16384,0.04206080039342244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,2560,0.024181334177652995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,16384,0.19825812975565593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,51200,0.586619758605957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,12288,0.03398400147755941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,16384,0.11055146853129069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,10240,0.025645865996678667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,12288,0.15272000630696614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,10240,0.06967466672261556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,10240,0.13012693723042806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,8192,0.021747199694315593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,8192,0.10606613159179687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,7168,0.018227199713389076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,1024,0.01270080010096232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,768,0.010177066922187806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,7168,0.09382399717966715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,6144,0.016684800386428833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,6144,0.08034559885660807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,256,0.007166933516661327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,6144,0.04398399988810221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,5120,0.012081066767374676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,256,2048,0.020037333170572914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,5120,0.06531093517939249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,5120,0.037690667311350505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,4096,0.011556266744931539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,3584,0.01076693336168925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,4096,0.04976106484731038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,3584,0.041866668065388996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,3584,0.026706133286158246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,3072,0.009348266323407491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,3072,0.03538666566212972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,3072,0.023324799537658692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,2560,0.008682666222254436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,2560,0.0314848005771637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,2560,0.02070080041885376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,2048,0.007393066585063934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,2048,0.027797333399454754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,1536,0.006692266464233399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,1536,0.023848533630371094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,1536,0.014230400323867798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,1024,0.005117866893609365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,1024,0.020910932620366415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,51200,0.32437121073404945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,768,0.00452159990866979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,768,0.019564799467722573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,768,0.009567999839782714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,12288,0.0836949348449707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,512,0.003972266614437103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,512,0.008260266482830047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,512,0.018449066082636516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,256,0.003492266684770584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,256,0.01608746647834778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,256,0.007228800157705943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,128,0.003206400076548258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,128,0.015745066603024802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,64,0.003078400095303853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,8192,0.05726933479309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,64,0.01553600033124288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,128,32,0.003067733347415924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,128,32,0.015211733182271323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,65536,0.14424746831258137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,7168,0.050329601764678954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,51200,0.11535786787668864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,16384,0.039741865793863934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,16384,0.19774932861328126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,4096,0.031149866183598836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,65536,0.7475893020629882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,12288,0.030932267506917317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,51200,0.5862666447957356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,10240,0.027558400233586626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,8192,0.022920533021291097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,12288,0.15165333747863768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,7168,0.01497066617012024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,10240,0.12845546404520672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,8192,0.10392426649729411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,6144,0.012349866827329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,5120,0.010759466886520385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,4096,0.009506133198738099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,6144,0.07849706808725992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,5120,0.06385279893875122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,3584,0.008357333143552144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,2048,0.017037866512934367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,4096,0.04840000073115031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,3072,0.00790293316046397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,2560,0.006855466465155284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,2048,0.00631039987007777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,3072,0.03511573473612468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,2560,0.031123199065526325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,1536,0.005530666808287303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,2048,0.026997333765029906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,1536,0.024475733439127602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,1024,0.004545066754023234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,768,0.004058666775623957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,1024,0.02050666610399882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,512,0.0036469332873821257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,768,0.01904319922129313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,256,0.003253333270549774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,1024,0.011363200346628825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,128,0.0030879999200503034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,256,0.01624853312969208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,64,0.002942933390537898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,128,0.015478400389353433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,64,32,0.002942933390537898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,64,0.015034666657447815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,32,0.015056000153223673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,65536,0.14235626856486003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,51200,0.1131167968114217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,16384,0.04178560177485148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,12288,0.0329312006632487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,16384,0.19721387227376302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,10240,0.026172800858815508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,12288,0.15091946919759114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,8192,0.022670932610829673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,10240,0.12762773036956787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,51200,0.5858346939086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,65536,0.7473066965738933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,7168,0.09221546649932862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,7168,0.01609386702378591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,6144,0.012436266740163167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,5120,0.011382399996121725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,7168,0.09198186397552491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,6144,0.07958186467488607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,4096,0.009810133775075277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,5120,0.06414080063501994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,3584,0.009027199943860371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,4096,0.04807146787643433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,3072,0.007924266656239827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2817,128,128,0.006066133578618368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,3584,0.04121919870376587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,3072,0.035122132301330565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,2560,0.029874134063720702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,1536,0.005245866874853769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,2048,0.02753173311551412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,512,0.01801919937133789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,1024,0.004347733159859975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,1536,0.02361066738764445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,768,0.0040554667512575785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,1024,0.021204266945521036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,512,0.0035861333211263022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,768,0.018883200486501057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,256,0.0031669333577156065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,512,0.017629865805308023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,128,0.00297173336148262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,256,0.01599253316720327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,64,0.0028704000016053517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,128,0.015409066279729208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,32,0.0028949332733949023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,64,0.0151829332113266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,32,0.014803199966748556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,32,8192,0.10399253368377685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2817,64,3584,0.04089279969533284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,2560,0.007269333302974701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2817,32,2048,0.006235733131567637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,16384,4.535393269856771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,12288,3.3908330281575525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,16384,5.27495371500651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,12288,6.207976786295573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,16384,8.165162658691406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,10240,2.7842740376790363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,12288,3.903924306233724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,10240,3.499909210205078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,10240,5.128107706705729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,8192,2.320903523763021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,8192,2.6024831136067705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,8192,4.3149470011393225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,7168,1.9618122100830078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,51200,14.223365275065103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,7168,4.425114440917969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,6144,2.0726858774820967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,6144,2.150487518310547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,6144,3.767963663736979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,5120,1.544104512532552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,4096,1.1512650807698568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,5120,2.7468299865722656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,7168,2.393396250406901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,4096,2.61996587117513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,3584,0.9763029098510743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,4096,1.5087103525797525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,3584,1.2096853892008463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,51200,18.90982666015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,3584,1.8374261220296222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,3072,0.8914976119995117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,3072,1.1312245686848958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,3072,1.7128395080566405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,2560,0.7153674443562825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,2560,1.2893450419108072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,2560,0.8738751729329428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,5120,1.74400634765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,2048,0.6453450520833333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,2048,1.0530111948649088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,1536,0.46883198420206706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,1536,0.8131754557291666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,2048,0.7863690694173177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,1536,0.6067712148030598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,1024,0.36198933919270837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,1024,0.607754643758138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,768,0.45571521123250325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,1024,0.5236042658487956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,512,0.296395746866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,768,0.32319040298461915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,512,0.2190080006917318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,256,0.17287999788920086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,768,0.41732479731241867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,128,0.15556373596191406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,512,0.4170677185058594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,128,0.16141653060913086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,64,0.14480427106221516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,32,0.14600000381469727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,128,0.2717952092488607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,32,0.1535029411315918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,256,0.19058559735616049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,65536,64,0.1578229268391927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,65536,256,0.2725023905436198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,65536,51200,26.056207275390626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,16384,3.5875691731770836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,16384,6.585400390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,51200,10.607635498046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,16384,4.225320434570312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,65536,15.044611612955729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,12288,3.175153096516927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,51200,14.425938924153646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,12288,4.799029541015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,12288,3.0452011108398436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,10240,2.293945566813151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,10240,2.768834177652995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,10240,4.1035924275716145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,8192,3.5501439412434896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,8192,2.2304128011067705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,65536,20.55679931640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,7168,1.6203744252522785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,51200,21.472721354166666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,7168,2.8325355529785154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,6144,1.2917887369791665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,6144,2.58671137491862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,5120,1.0906251271565757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,6144,1.616106669108073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,5120,1.3510964711507163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,8192,2.1344009399414063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,5120,2.152220916748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,4096,0.9072575887044272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,4096,1.1161930084228515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,4096,1.6867764790852864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,3584,0.7957674662272136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,3584,1.0123221079508462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,3072,0.7412042617797852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,7168,1.8510005950927735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,3072,1.2772767384847006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,2560,0.6099669138590496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,3072,0.8091893513997397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,2560,1.0513013203938804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,2560,0.72401917775472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,2048,0.50043732325236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,2048,0.8935125350952149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,1536,0.4096320152282715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,1024,0.29188693364461266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,1536,0.6299114863077799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,1024,0.43107093175252276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,1536,0.5126869201660156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,3584,1.5250432332356771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,768,0.24705813725789388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,512,0.24298133850097656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,768,0.33591785430908205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,512,0.17648213704427082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,768,0.36795199712117516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,65536,26.731416829427083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,256,0.16883734067281086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,128,0.1257472038269043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,256,0.21602133115132652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,256,0.1590186595916748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,512,0.2858602523803711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,64,0.11593706607818603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,64,0.12770453294118245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,128,0.21182826360066734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,51200,32,0.11392107009887695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,32,0.12674026489257811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,51200,128,0.14621225992838544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,2048,0.5679082870483398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,51200,1024,0.33935359319051106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,16384,1.1911530812581381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,51200,3.8429290771484377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,16384,2.223213958740234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,51200,4.3882293701171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,65536,4.929027303059896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,16384,1.3951807657877604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,12288,0.9467466354370118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,12288,1.5545492808024088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,65536,5.887583923339844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,10240,0.7883701324462891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,10240,0.8609024047851562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,10240,1.274716822306315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,51200,6.362644449869792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,8192,1.069037882486979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,8192,0.6063434600830078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,8192,0.6649258931477864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,7168,0.5457578659057617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,7168,0.590656026204427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,6144,0.4727007865905762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,12288,0.9999071756998698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,7168,0.9655349095662435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,6144,0.5357983907063801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,6144,0.8346527735392252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,5120,0.6248544057210286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,5120,0.3945525487263998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,4096,0.3474261283874512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,5120,0.42180693944295244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,4096,0.4954389254252116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,3584,0.29053440093994143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,3584,0.3131701469421387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,3584,0.49727894465128586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,3072,0.26538027127583824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,3072,0.4022101402282715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,3072,0.28743254343668617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,65536,8.570332845052082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,2560,0.23274666468302407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,2560,0.3127605438232422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,2048,0.2569023927052816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,2560,0.270411745707194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,2048,0.1940661271413167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,2048,0.2157503922780355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,1536,0.2163306713104248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,1536,0.1464032014211019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,1024,0.14959680239359538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,1536,0.18143787384033203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,1024,0.1421557267506917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,768,0.09315306345621745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,768,0.12750399907430013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,512,0.08385173479715982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,768,0.1282549301783244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,512,0.07698559761047363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,256,0.06006613175074259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,512,0.09791466395060221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,256,0.06092906792958578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,128,0.04399786790211995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,256,0.08915839989980062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,64,0.03942293326059977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,128,0.058641068140665685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,128,0.07093866666158041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,16384,32,0.040429866313934325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,64,0.05588266849517822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,32,0.05526613394419352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,16384,1024,0.11457599798838299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,16384,4096,0.3337781270345052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,51200,2.9096031188964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,16384,1.5471295674641927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,16384,0.882587750752767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,65536,3.9819625854492187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,12288,0.6823221206665039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,12288,1.1057963053385416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,51200,4.706827799479166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,12288,0.8380693435668946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,10240,0.594712511698405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,10240,0.9395392100016275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,10240,0.6173760096232097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,8192,0.47500267028808596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,16384,1.0648746490478516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,8192,0.8428010940551758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,8192,0.5350261052449544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,7168,0.41195093790690107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,7168,0.6561365127563477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,7168,0.4205301284790039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,6144,0.3949365297953288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,65536,6.468250528971355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,6144,0.5735455830891927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,5120,0.2872821489969889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,5120,0.3362677256266276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,4096,0.2603797276814779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,5120,0.4722282727559407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,4096,0.4025578816731771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,4096,0.2674015998840332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,3584,0.2303786595662435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,3584,0.3258570671081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,3584,0.2494954744974772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,3072,0.28690239588419597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,3072,0.19594987233479816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,3072,0.20326080322265624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,2560,0.24491519927978517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,2048,0.18299840291341146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,2048,0.13762240409851073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,2048,0.16687893867492676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,1536,0.141592534383138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,65536,4.323272705078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,51200,3.1755551656087238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,1536,0.11283520062764485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,1024,0.10088319778442383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,1536,0.13757120768229167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,768,0.08159253597259522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,768,0.07382933298746744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,1024,0.09491946697235107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,768,0.09520959854125977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,1024,0.12476053237915039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,512,0.05990720192591349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,256,0.04016853173573812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,512,0.05913706620534261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,512,0.0813098669052124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,256,0.057378133138020836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,128,0.03421546618143718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,6144,0.3764447848002116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,128,0.045227734247843425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,64,0.03253333369890849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,128,0.0574282685915629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,64,0.046410667896270755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,12288,32,0.031779199838638306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,32,0.04668800036112468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,12288,2560,0.17012054125467937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,2560,0.1712000052134196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,51200,2.527424112955729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,12288,256,0.05682346820831299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,65536,3.248422495524088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,51200,2.787378184000651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,16384,0.8008234659830729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,65536,3.660362752278646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,16384,0.8298976262410482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,51200,4.0684862772623696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,12288,0.652243169148763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,12288,0.6431936264038086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,12288,0.9932650883992513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,10240,0.5634154637654623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,10240,0.804868253072103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,10240,0.5433888117472331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,8192,0.44040638605753585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,8192,0.682049051920573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,8192,0.44368851979573565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,65536,5.219088236490885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,7168,0.4086474736531575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,7168,0.5466037114461263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,7168,0.3891146659851074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,6144,0.34555625915527344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,6144,0.5044586817423503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,5120,0.2825514793395996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,5120,0.4102805455525716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,5120,0.2978314717610677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,6144,0.3566677411397298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,4096,0.32411092122395835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,4096,0.24213013648986817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,4096,0.23988053003946938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,3584,0.21217813491821289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,3584,0.29318081537882484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,3072,0.18160319328308105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,3584,0.23707092603047691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,2560,0.20396800041198732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,2560,0.16135573387145996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,2048,0.1673408031463623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,2560,0.16740479469299316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,2048,0.12826666831970215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,1536,0.13059306939442952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,2048,0.1410645325978597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,1536,0.10754773616790772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,1536,0.11653546492258708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,1024,0.0930613358815511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,1024,0.08099413712819417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,1024,0.09534613291422525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,768,0.07017813523610433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,16384,1.275551986694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,512,0.05691946744918823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,256,0.036637866497039796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,512,0.05442026853561401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,512,0.061033598581949865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,256,0.044616532325744626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,128,0.029179733991622925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,128,0.04210773309071859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,128,0.04731626510620117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,64,0.04191146691640218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,32,0.027405865987141925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,32,0.04256106615066528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,3072,0.24696853955586753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,10240,768,0.06983146667480469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,10240,64,0.027059199412663777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,3072,0.17139946619669597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,768,0.07239253520965576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,10240,256,0.050213332970937094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,65536,2.723768615722656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,51200,2.136840565999349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,65536,2.9460405985514324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,16384,0.671617062886556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,16384,0.960483169555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,16384,0.6871349334716796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,12288,0.5503317515055339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,12288,0.5294538815816243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,51200,3.2850313822428383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,65536,4.217650095621744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,12288,0.7774752298990886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,10240,0.46685333251953126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,10240,0.6725119908650716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,51200,2.2728042602539062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,10240,0.43660907745361327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,8192,0.3803914705912272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,7168,0.32329174677530925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,8192,0.5351114908854167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,7168,0.3352575937906901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,7168,0.4753877321879069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,6144,0.4299296061197917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,6144,0.3051103909810384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,5120,0.24482986132303872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,5120,0.2389450709025065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,5120,0.31291627883911133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,4096,0.27440106074015297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,4096,0.21262826919555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,4096,0.2016138712565104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,3584,0.17535360654195148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,3584,0.23265600204467773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,3072,0.1992618719736735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,3072,0.15517652829488118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,2560,0.16606826782226564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,2560,0.13387093544006348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,2560,0.13265493710835774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,2048,0.13887467384338378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,2048,0.11054720083872478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,2048,0.1113162676493327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,1536,0.10179839928944905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,1536,0.08858239650726318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,1024,0.07373226483662923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,1024,0.06896426677703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,1024,0.06933226585388183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,768,0.056898132960001624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,768,0.05966933170954386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,512,0.043171199162801106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,8192,0.3567328135172526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,512,0.047858134905497236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,256,0.028654932975769043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,512,0.050171732902526855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,256,0.03829653263092041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,256,0.0414133350054423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,128,0.02304960091908773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,128,0.036850134531656906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,128,0.03768959840138753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,64,0.022101332743962608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,64,0.036549333731333414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,32,0.023822933435440063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,8192,32,0.03691093524297078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,8192,768,0.057214931646982825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,6144,0.26061760584513344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,3584,0.1572490692138672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,3072,0.13923519452412922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,8192,1536,0.08484799861907959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,65536,2.6326751708984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,16384,0.6604938507080078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,16384,0.9608394622802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,51200,2.0272650400797527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,65536,4.0653226216634115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,12288,0.541709836324056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,12288,0.7208394368489583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,10240,0.4247402509053548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,10240,0.6037226359049479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,51200,3.2278976440429688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,8192,0.4884895960489909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,8192,0.3481482823689779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,7168,0.30520426432291664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,7168,0.4365258534749349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,6144,0.37172266642252605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,6144,0.28879146575927733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,16384,0.6729557037353515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,65536,3.1173492431640626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,5120,0.30683838526407875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,5120,0.22996692657470702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,51200,2.379383341471354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,4096,0.24338879585266113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,12288,0.5363765080769857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,10240,0.47355200449625656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,4096,0.19163947105407714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,3584,0.21432107289632163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,3584,0.16323839823404948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,3072,0.18494505882263185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,3072,0.13632426261901856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,2560,0.15507946014404297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,8192,0.3603637377421061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,2560,0.11488640308380127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,2048,0.1249888022740682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,2048,0.09719786643981934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,7168,0.3217599868774414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,1536,0.0976149320602417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,1536,0.07961599826812744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,6144,0.2670432090759277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,1024,0.0645365317662557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,1024,0.06324373483657837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,5120,0.2224031925201416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,768,0.049965866406758624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,768,0.052585601806640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,4096,0.18357332547505695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,512,0.03829973141352336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,3584,0.1590816020965576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,512,0.044368000825246175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,256,0.025987199942270917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,256,0.03654293219248454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,3072,0.1344480037689209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,128,0.021617066860198975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,128,0.033190399408340454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,2560,0.1233365297317505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,64,0.01927466591199239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,64,0.033752532800038655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,7168,32,0.019642666975657145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,7168,32,0.03391040166219075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,2048,0.09922880331675211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,1536,0.07535466353098551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,1024,0.06124586661656698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,768,0.047931734720865884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,65536,2.1214197794596354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,512,0.03858986695607503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,256,0.032662399609883624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,7168,128,0.027369600534439088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,16384,0.681384531656901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,65536,3.2004224141438806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,16384,0.530842653910319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,16384,0.49434668223063155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,51200,2.484234619140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,12288,0.5738069534301757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,12288,0.42280108133951827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,51200,1.6307231903076171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,10240,0.4642773310343425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,10240,0.36816533406575525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,8192,0.35755945841471354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,10240,0.3592512130737305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,8192,0.2868074735005697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,8192,0.2892512003580729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,7168,0.23551467259724937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,7168,0.2715914726257324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,7168,0.3407391866048177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,6144,0.28275839487711585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,6144,0.21712640126546226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,6144,0.2197663942972819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,5120,0.24510505994160972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,5120,0.1970901330312093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,5120,0.16966932614644367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,4096,0.1847434679667155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,4096,0.16678080558776856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,4096,0.1535861333211263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,3584,0.15942506790161132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,3584,0.12992746829986573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,3072,0.1358847935994466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,3584,0.14674347241719563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,12288,0.39775040944417317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,3072,0.11953279972076417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,2560,0.11553706328074138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,3072,0.11542932987213135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,2560,0.09415040016174317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,2560,0.11172373294830322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,2048,0.09647040367126465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,2048,0.08471786975860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,1536,0.07367146809895833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,1024,0.054248531659444175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,51200,1.717350387573242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,1024,0.05799680153528849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,1024,0.05571626822153727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,768,0.04258986711502075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,768,0.04916906754175822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,768,0.04675413370132446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,512,0.03200213313102722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,256,0.02193280061086019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,512,0.04120106697082519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,512,0.039603201548258464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,128,0.018051199118296304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,256,0.03259200056393941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,128,0.030165332555770873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,128,0.030061866839726763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,64,0.015425067146619162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,32,0.01623253325621287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,64,0.0302837332089742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,65536,2.2559913635253905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,32,0.03081600069999695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,6144,1536,0.07310400009155274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,256,0.03371200164159139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,65536,1.9509525299072266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,65536,1.9905343373616535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,6144,2048,0.08821760018666586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,6144,1536,0.06565226713816324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,16384,0.44167254765828445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,16384,0.7043882369995117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,65536,2.873816426595052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,16384,0.5160895983378093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,51200,1.530894978841146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,51200,2.1784276326497394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,51200,1.4909589131673178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,12288,0.4699434598286946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,12288,0.386025587717692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,12288,0.36176214218139646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,10240,0.29146881103515626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,10240,0.32887252171834314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,10240,0.4319423993428548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,8192,0.3435253461201986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,8192,0.3067647933959961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,8192,0.2555573304494222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,7168,0.27818028132120765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,7168,0.24934612909952797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,7168,0.20940160751342773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,6144,0.24081813494364418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,5120,0.21954132715861002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,5120,0.15749972661336262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,5120,0.18628586133321126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,4096,0.1678122679392497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,4096,0.12661653359731037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,3584,0.15110079447428387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,3072,0.12938026587168378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,3584,0.13186026414235433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,3584,0.12432640393575031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,3072,0.11196479797363282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,3072,0.10560213724772136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,2560,0.11049706935882568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,2048,0.08880533377329508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,2560,0.09553919633229574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,2560,0.10137813091278076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,2048,0.08724479675292969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,1536,0.06934186617533365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,1024,0.05145279963811239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,1536,0.07151040236155191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,1024,0.05431040128072103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,768,0.04296319882074992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,768,0.04484479824701945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,768,0.04660906791687012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,512,0.033079467217127484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,512,0.038881067434946695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,512,0.035626665751139326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,256,0.021783467133839926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,256,0.03231146732966105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,128,0.01676266590754191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,6144,0.2182624022165934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,4096,0.1430005391438802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,128,0.029250133037567138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,64,0.014500266313552857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,128,0.026338134209314985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,5120,32,0.01420799990495046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,64,0.028728532791137695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,5120,32,0.027958399057388304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,256,0.029127466678619384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,6144,0.17703893979390461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,2048,0.0719808022181193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,1536,0.060133334000905356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,5120,1024,0.04784533182779948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,65536,1.6475755055745442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,65536,2.1780469258626303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,16384,0.47336746851603195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,16384,0.4356543858846028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,16384,0.4911498705546061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,12288,0.3836661338806152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,12288,0.33010772069295247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,12288,0.33952960968017576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,51200,1.2812447865804037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,10240,0.3106997489929199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,10240,0.30579732259114584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,51200,1.4724447886149088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,8192,0.24099946022033691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,8192,0.22438294092814126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,8192,0.27049919764200847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,7168,0.22290347417195638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,7168,0.20822292963663735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,6144,0.18693440755208332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,6144,0.18535680770874025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,5120,0.16158612569173175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,5120,0.14419946670532227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,4096,0.1258133331934611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,4096,0.12423573335011799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,3584,0.10884160200754803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,3584,0.10981333255767822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,51200,1.6685600280761719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,3072,0.09500053723653158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,3072,0.09701013565063477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,65536,1.899337641398112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,2560,0.08903679847717286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,2560,0.09663253625233968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,2560,0.08429546356201172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,2048,0.06734826564788818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,2048,0.0739402691523234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,10240,0.2932917277018229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,1536,0.049407998720804855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,2048,0.06745173136393229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,1536,0.05152320067087809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,1536,0.05877759853998819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,1024,0.03513280153274536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,1024,0.04654293457667033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,7168,0.20304746627807618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,768,0.03218986590703328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,768,0.04047040144602458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,6144,0.17201174100240071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,768,0.03276159962018331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,512,0.025178666909535723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,512,0.033920001983642575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,512,0.026428800821304322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,5120,0.1483338673909505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,256,0.02815679907798767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,4096,0.1189184029897054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,128,0.02510720094045003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,64,0.01162986656030019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,64,0.025618133942286174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,32,0.01211840013662974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,3584,0.10468693574269612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,3072,0.08887999852498372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,256,0.01809920072555542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,4096,128,0.013769599795341491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,4096,32,0.025733333826065064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,65536,1.8578837076822918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,65536,1.549283218383789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,51200,1.3422091166178385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,16384,0.4259562810262044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,1024,0.040749867757161454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,65536,1.5896415710449219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,256,0.02174293398857117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,51200,1.2017759958902994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,12288,0.32000745137532555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,4096,128,0.017938133080800375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,12288,0.33327039082845056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,16384,0.4215488115946452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,12288,0.2884853363037109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,10240,0.26651199658711755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,10240,0.25536212921142576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,8192,0.23921066919962564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,8192,0.19167146682739258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,8192,0.21644585927327475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,7168,0.20195199648539225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,7168,0.18305813471476237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,6144,0.16242772738138836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,6144,0.16312959988911946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,5120,0.13713493347167968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,4096,0.11115093231201172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,5120,0.13383572896321613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,4096,0.10713067054748535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,6144,0.16237227121988934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,3584,0.09691946506500244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,3584,0.0964458703994751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,3584,0.0897877295811971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,3072,0.08425599733988444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,3072,0.08606826464335124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,3072,0.075654403368632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,2560,0.07144746780395508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,16384,0.37341972986857097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,2560,0.07552320162455241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,2560,0.06752320130666098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,2048,0.058431998888651526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,51200,1.209115727742513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,2048,0.05681066513061524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,2048,0.06499626636505126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,10240,0.25518293380737306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,1536,0.05345600048700968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,1024,0.032126933336257935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,1536,0.05187520186106363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,1024,0.04432959953943889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,768,0.02619839906692505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,1024,0.03537813425064087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,512,0.019792000452677407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,768,0.029284266630808513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,768,0.0382261315981547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,7168,0.17085119883219402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,256,0.014578133821487427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,512,0.02940160036087036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,128,0.011678933103879293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,256,0.02617173393567403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,128,0.02374826669692993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,64,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,64,0.02397866646448771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,5120,0.12688746452331542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3584,32,0.011834667126337687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,4096,0.10050880114237468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3584,32,0.024392533302307128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,256,0.020863999923070274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,65536,1.4524383544921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,65536,1.3642965952555337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,51200,1.0515647888183595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,51200,0.9235434850056967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,1536,0.044097065925598145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,51200,1.0961429595947265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,16384,0.3425504048665365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,16384,0.39724906285603845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,12288,0.25821866989135744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,512,0.023306665817896526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,16384,0.3164448102315267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,12288,0.22217280069986978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,12288,0.2725599924723307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,10240,0.2181386629740397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3584,128,0.015846400459607442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,8192,0.1796618620554606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,10240,0.22930986086527505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,10240,0.18799786567687987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,8192,0.18438720703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,7168,0.16076159477233887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,6144,0.13217066923777263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,7168,0.14618879954020184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,6144,0.11768213113149006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,6144,0.13992533683776856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,5120,0.11090772946675617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,5120,0.10393493175506592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,5120,0.12084373633066814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,4096,0.09019839763641357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,4096,0.09769813219706217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,4096,0.08050026893615722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,3584,0.08053119977315268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,3072,0.06809813181559245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,3584,0.08913173675537109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,3072,0.07704532941182454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,2560,0.057726931571960446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,2560,0.05511786540349325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,2048,0.04729173183441162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,2048,0.047515734036763506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,2048,0.056536531448364256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,1536,0.0368010679880778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,1536,0.045932801564534505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,1536,0.039692799250284835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,1024,0.026305067539215087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,1024,0.03154559930165608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,1024,0.03526186545689901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,768,0.02149440050125122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,768,0.031092266241709392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,512,0.016547200083732606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,512,0.026693334182103474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,512,0.022502400477727256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,65536,1.2255626678466798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,256,0.011526399850845337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,7168,0.1556320031483968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,128,0.008888533711433411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,256,0.018857600291570027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,256,0.02704746723175049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,128,0.017063466707865398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,128,0.021333332856496176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,64,0.008932266632715862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,3072,32,0.008248533308506011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,64,0.0209824005762736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,32,0.02277759909629822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,3072,2560,0.06858879725138346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,65536,1.3296469370524089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,65536,1.2803627014160157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,8192,0.1508021354675293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,3072,0.06371946732203165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,3584,0.07216640313466391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,51200,1.037499745686849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,16384,0.29482666651407874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,51200,1.0186815897623698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,16384,0.3306175867716471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,12288,0.22361812591552735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,3072,768,0.027134933074315387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,12288,0.25793066024780276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,12288,0.2379456043243408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,10240,0.1897119998931885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,10240,0.21202452977498373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,8192,0.15244693756103517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,8192,0.18672746022542316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,7168,0.1348373254140218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,7168,0.16517972946166992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,6144,0.11932266553243001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,6144,0.14029760360717775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,5120,0.0968565305074056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,5120,0.11514773368835449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,5120,0.10039467016855877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,4096,0.07819519837697347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,4096,0.09257067044576009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,65536,1.2784661610921224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,3584,0.07758613427480063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,3584,0.08773760000864664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,51200,0.9518549601236979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,3072,0.06194666624069214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,16384,0.2922613461812337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,3072,0.07821333408355713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,2560,0.053046401341756186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,2560,0.06702079772949218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,10240,0.1903605302174886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,8192,0.15204159418741864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,2048,0.04204800128936768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,2048,0.05268266598383585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,2048,0.0449343999226888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,1536,0.03216000000635783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,1536,0.041553068161010745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,1024,0.022833067178726196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,7168,0.1353973388671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,1024,0.03337920109430949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,768,0.019002666076024376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,768,0.03011946678161621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,6144,0.11660906473795574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,512,0.014778666694959006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,512,0.026855466763178508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,4096,0.08051199913024902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,256,0.011197866996129353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,256,0.02443199952443441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,3584,0.07064747015635173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,128,0.009746133287747701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,128,0.02068159977595011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,128,0.012230400244394939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,64,0.009008000294367473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,64,0.0206496000289917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2560,32,0.009577600161234538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2560,32,0.020734934012095134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,3072,0.06021866798400879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,2560,0.05498133500417074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,1536,0.03470293283462524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,1024,0.02858346700668335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,65536,1.0133973439534505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,768,0.021580799420674642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,65536,1.1415519714355469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,512,0.017527467012405394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,51200,0.7357418696085612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,16384,0.24756906827290853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,16384,0.29749654134114584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,12288,0.1803541342417399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,12288,0.22889493306477865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,10240,0.16558292706807454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,51200,0.8983829498291016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,10240,0.19317760467529296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2560,256,0.014500266313552857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,8192,0.12075626850128174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,7168,0.10646719932556152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,8192,0.1573919932047526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,7168,0.1387370745340983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,6144,0.09332053661346436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,6144,0.12202773094177247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,65536,1.135661824544271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,5120,0.08051093419392905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,16384,0.28390719095865885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,12288,0.20839893023173012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,5120,0.10807039737701415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,51200,0.8886079788208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,4096,0.06395946741104126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,4096,0.08914346694946289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,3584,0.055027198791503903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,10240,0.18265172640482585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,3584,0.07991360028584799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,3584,0.06497600078582763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,3072,0.047652268409729005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,8192,0.14341120719909667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,2560,0.0406058669090271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,3072,0.06981866359710694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,2560,0.05479360024134318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,2560,0.05458026727040609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,2048,0.03302613298098246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,2048,0.046555733680725096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,7168,0.12612799803415936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,1536,0.026359466711680095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,1536,0.03558613459269206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,1024,0.018627200524012247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,1024,0.02898453275362651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,1024,0.02417280077934265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,6144,0.10914026896158854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,768,0.01440000037352244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,768,0.025278933842976886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,512,0.010913067062695821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,512,0.023153066635131836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,5120,0.09443626403808594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,4096,0.07528639634450277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,256,0.007621333499749501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,256,0.02056000034014384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,3072,0.055371733506520596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,128,0.00664213349421819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,128,0.02143893241882324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,64,0.008493866523106892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,64,0.021262933810551964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,2048,0.040277334054311116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,2048,32,0.008910933136940002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,2048,32,0.02142613331476847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,1536,0.030817067623138426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,768,0.019460266828536986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,65536,0.6946687698364258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,51200,0.514298661549886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,512,0.015833600362141927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,16384,0.18489279747009277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,65536,1.0077856063842774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,16384,0.2648362636566162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,12288,0.12751680215199787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,51200,0.6236522674560547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,12288,0.20193173090616862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,10240,0.11765226523081462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,10240,0.17029760678609213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,8192,0.08762239615122477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,8192,0.13950079282124836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,8192,0.09904320240020752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,7168,0.07741866906483968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,7168,0.12469013532002766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,128,0.011327999830245971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,51200,0.7950069427490234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,2048,256,0.013692800203959146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,6144,0.0662506659825643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,5120,0.05592853228251139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,6144,0.07629226843516032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,6144,0.10858773390452067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,4096,0.04530666669209798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,5120,0.09225707054138184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,4096,0.052634668350219724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,4096,0.07440959612528483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,3584,0.04043200016021729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,3584,0.06573333342870077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,3072,0.034701867898305254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,3072,0.05763520002365112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,65536,0.7210698445638021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,2560,0.032270934184392294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,2560,0.051785600185394284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,2560,0.040158931414286295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,16384,0.18620905876159669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,2048,0.026848000288009644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,2048,0.04017920096715291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,1536,0.020009599129358926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,1536,0.03309440016746521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,12288,0.14441067377726238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,1024,0.014838400483131408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,1024,0.027430399258931475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,1024,0.0203167994817098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,768,0.012217600146929424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,10240,0.12444586753845215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,768,0.02443199952443441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,512,0.009739733735720317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,512,0.02179946700731913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,512,0.01502293348312378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,256,0.007417599856853485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,256,0.019768534104029338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,256,0.012585600217183432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,128,0.006274133423964183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,128,0.018267732858657838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,7168,0.08789973258972168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,64,0.005549866457780202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,64,0.018067200978597008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1536,32,0.005760000149408976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1536,32,0.018433066209157307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,5120,0.06476586659749349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,65536,0.46770985921223956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,3584,0.04694613218307495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,3072,0.041740798950195314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,51200,0.3659893353780111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,65536,0.9100394566853842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,16384,0.12101973692576092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,2048,0.030850134293238324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,16384,0.23958187103271483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,1536,0.02502826650937398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,51200,0.7116383870442708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,12288,0.09191040198008218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,768,0.01734293301900228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,10240,0.07579733530680338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,10240,0.154093869527181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1536,128,0.01122773289680481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,8192,0.06044160127639771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,8192,0.12659413019816082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,12288,0.18087040583292643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,7168,0.05347306728363037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,7168,0.11301759878794353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,6144,0.04597226778666179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,6144,0.0982208013534546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,12288,0.15628053347269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,5120,0.0400874654452006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,5120,0.0835914691289266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,65536,0.8039072036743165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,4096,0.0322655995686849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,4096,0.06657919883728028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,10240,0.1316159963607788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,51200,0.6286656061808269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,3584,0.05883200168609619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,16384,0.20702187220255533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,3072,0.024437334140141806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,3072,0.04923839966456096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,3584,0.028427733977635698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,2560,0.02104319930076599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,8192,0.10607039928436279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,2560,0.04107413291931152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,7168,0.09350612958272299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,2048,0.017780266205469766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,2048,0.03519253333409627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,1536,0.01440000037352244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,1536,0.030083199342091877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,6144,0.08038933277130127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,1024,0.01076693336168925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,1024,0.0247381329536438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,3072,0.041819731394449874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,4096,0.05589973529179891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,768,0.00909546713034312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,768,0.02260800004005432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,512,0.007589333256085714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,512,0.02037013371785482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,5120,0.06850132942199708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,256,0.006086400151252747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,256,0.01862186590830485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,1536,0.022025599082310995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,128,0.005037866532802582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,128,0.01714986761411031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,3584,0.04835306803385417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,64,0.004574933151404063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,64,0.017204266786575318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,1024,32,0.004743466774622599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,1024,32,0.01688106656074524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,2560,0.03567146857579549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,65536,0.3539711952209473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,1024,0.01764799952507019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,2048,0.029099732637405396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,51200,0.273854923248291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,65536,0.8461887995402018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,16384,0.09030400117238363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,16384,0.22187093098958335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,51200,0.6650815963745117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,12288,0.06925333340962728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,12288,0.1720234711964925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,65536,0.5115615844726562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,256,0.00993066628774007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,10240,0.057866664727528896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,10240,0.0856725295384725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,8192,0.046773334344228104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,10240,0.14800960222880047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,128,0.008363733688990276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,7168,0.04098026752471924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,51200,0.39969278971354166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,8192,0.11940053304036458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,6144,0.035796264807383224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,7168,0.10639039675394694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,512,0.011885866522789001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,6144,0.09255786736806235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,16384,0.13385920524597167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,5120,0.030397866169611615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,4096,0.024870399634043375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,5120,0.046087467670440675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,5120,0.07780799865722657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,3584,0.021951999266942343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,4096,0.061586133639017736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,3584,0.05283626715342203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,3072,0.019092265764872232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,12288,0.10231573581695556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,3072,0.04536426862080892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,1024,768,0.014526933431625366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,2560,0.016429866353670754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,2560,0.03820799986521403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,2560,0.02601813276608785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,2048,0.013670399785041809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,2048,0.032953600088755294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,2048,0.021591466665267945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,1536,0.011275733510653179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,1536,0.027411200602849323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,1536,0.018045866489410402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,1024,0.008644266923268636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,1024,0.023217066129048666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,8192,0.07082453568776449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,768,0.00769706666469574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,7168,0.0620746652285258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,768,0.021627734104792275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,768,0.012733866771062216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,512,0.006428800026575724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,512,0.01106666624546051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,6144,0.053795198599497475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,256,0.005298133194446564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,256,0.017540266116460167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,4096,0.038738131523132324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,128,0.004520533482233683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,128,0.008102400104204814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,128,0.016874667008717856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,64,0.0041802664597829185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,768,32,0.004281599819660187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,32,0.01614720026652018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,3072,0.02975146571795146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,65536,0.24478933016459145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,1024,0.01479573349157969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,65536,0.8073770523071289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,512,0.01944213310877482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,16384,0.10432106653849285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,3584,0.034681598345438644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,16384,0.22915412584940592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,12288,0.08014506498972575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,12288,0.1753877321879069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,51200,0.19461867014567058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,10240,0.06775360107421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,10240,0.14947199821472168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,768,64,0.016432000199953715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,768,256,0.009399466713269551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,8192,0.05386026700337728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,8192,0.11464213530222575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,7168,0.048401065667470294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,7168,0.1016160011291504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,51200,0.6386698404947917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,6144,0.04143786827723185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,6144,0.0868117332458496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,12288,0.10817173322041829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,65536,0.5400266647338867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,5120,0.031838933626810714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,5120,0.07292693456013997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,4096,0.018309332927068076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,4096,0.05682453314463297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,16384,0.14313066800435384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,3584,0.01648853321870168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,3584,0.04832213322321574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,51200,0.42089494069417316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,3072,0.014659200112024942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,3072,0.04138559897740682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,10240,0.09120852947235107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,2560,0.012689066926638284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,2560,0.03515093326568604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,6144,0.05654079914093017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,7168,0.06594773530960082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,2048,0.01086293359597524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,2048,0.030906667311986286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,1536,0.009227733810742695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,1536,0.026359466711680095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,8192,0.07349013487497966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,1024,0.007277866701285045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,1024,0.022725333770116173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,3072,0.03017599980036418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,768,0.006450133522351582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,768,0.02057386636734009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,4096,0.04008640050888061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,512,0.005497600138187409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,512,0.0189194659392039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,1536,0.017246933778127034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,5120,0.04767466783523559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,256,0.004733866453170777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,256,0.017292799552281697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,128,0.0038880000511805216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,128,0.01634880006313324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,2560,0.025961599747339886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,64,0.0035455999275048576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,64,0.016286933422088624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,3584,0.035331201553344724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,512,32,0.003731200098991394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,512,32,0.016292267044385276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,65536,0.154694398244222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,65536,0.75829652150472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,1024,0.013783466815948487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,51200,0.12375679810841878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,2048,0.021955200036366782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,16384,0.048757334550221756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,16384,0.21278506914774575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,51200,0.5969482421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,12288,0.0394048015276591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,12288,0.1573098659515381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,128,0.006595199803511302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,10240,0.0465119997660319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,256,0.007944533228874206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,512,0.009310932954152425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,8192,0.03613866567611694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,10240,0.1313365300496419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,8192,0.10734186967213948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,512,768,0.011165866255760193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,7168,0.024293333292007446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,7168,0.09492692947387696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,65536,0.5246389389038086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,6144,0.018768000602722167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,6144,0.08546453317006429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,12288,0.10238933563232422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,5120,0.015426133076349893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,5120,0.06981546878814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,16384,0.13663039207458497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,4096,0.011703466375668842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,4096,0.05381120045979818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,51200,0.4103829383850098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,3584,0.010554666320482891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,3584,0.04566613435745239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,3072,0.014222932855288186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,3072,0.038780800501505536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,10240,0.08784426848093668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,2560,0.012009599804878235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,2560,0.03373119831085205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,8192,0.07112747033437093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,2048,0.010426666339238484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,2048,0.028757333755493164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,7168,0.06303040186564127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,1536,0.008613333106040955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,1536,0.025612799326578777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,6144,0.05450453360875448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,1024,0.0064640000462532045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,1024,0.02156053384145101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,3072,0.0285045325756073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,768,0.005564799904823304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,768,0.01967573364575704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,4096,0.03851199944814046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,512,0.004653866589069367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,512,0.018296533823013307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,5120,0.0457205335299174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,256,0.0038880000511805216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,3584,0.03340906699498494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,256,0.016428800423940022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,128,0.0034976000587145484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,128,0.015731199582417806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,2560,0.024387200673421226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,64,0.0033290666838486993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,1536,0.015983999768892924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,64,0.015969066818555196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,256,32,0.0034741332133611047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,256,32,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,65536,0.14799787203470866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,2048,0.0201365331808726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,51200,0.1173205296198527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,65536,0.7496927897135417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,51200,0.587666130065918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,16384,0.041444265842437746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,1024,0.012617599964141846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,16384,0.19978987375895182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,12288,0.0335477352142334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,12288,0.1525610605875651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,128,0.005692799886067709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,10240,0.02946346600850423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,256,0.006904533505439759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,10240,0.12846293449401855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,8192,0.024796799818674723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,768,0.010279466708501179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,256,512,0.008373333017031352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,8192,0.10482880274454753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,7168,0.09243199825286866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,7168,0.01798400084177653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,6144,0.013080533345540366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,6144,0.08024853070576986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,65536,0.483568000793457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,5120,0.011330133676528931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,5120,0.06508373419443766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,51200,0.3808160146077474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,4096,0.009673600395520527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,4096,0.04902400175730388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,16384,0.12615359624226888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,3584,0.008836266398429871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,3584,0.04128426710764567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,10240,0.08087999820709228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,3072,0.008042666812737782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,3072,0.03591466744740804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,8192,0.06629546483357748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,7168,0.05822933514912924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,2560,0.007276799778143566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,2560,0.0316213329633077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,2048,0.0063381334145863845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,2048,0.02722773353258769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,6144,0.05118826627731323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,1536,0.0055637334783871974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,1536,0.023970133066177367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,12288,0.09609920183817545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,1024,0.004795733094215393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,1024,0.020923733711242676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,5120,0.04292693138122559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,768,0.004378666480382284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,768,0.01908479928970337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,4096,0.03481706778208415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,512,0.003910399973392487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,512,0.017794134219487508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,3072,0.02643306652704875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,3584,0.03054080009460449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,256,0.0034965333839257562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,256,0.0162090669075648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,2560,0.022983467578887938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,128,0.0032405334214369455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,128,0.01554026703039805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,1536,0.015049599607785544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,2048,0.018651733795801796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,64,0.003018666555484136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,128,32,0.0031136001149813337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,64,0.015337600310643514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,128,32,0.01499626636505127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,65536,0.13766080538431805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,51200,0.10938453674316406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,65536,0.7401834487915039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,1024,0.012083199620246888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,51200,0.5837610880533854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,16384,0.037956265608469646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,12288,0.03026240070660909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,10240,0.02661973237991333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,12288,0.15119360287984213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,16384,0.19704532623291016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,8192,0.02209706703821818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,7168,0.01773866613705953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,10240,0.12804160118103028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,6144,0.011989333232243856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,8192,0.1041152000427246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,7168,0.09146880308787028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,5120,0.010265599687894185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,4096,0.00867306689421336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,6144,0.07800319989522299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,3584,0.00795839975277583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,5120,0.06416320006052653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,4096,0.048365867137908934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,3072,0.007365333537260692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,3584,0.04046613375345866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,2560,0.0069578667481740315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,3072,0.03507413466771443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,2048,0.006206933160622915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,2560,0.030852266152699787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,1536,0.005381333331267038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,2048,0.027013333638509114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,1024,0.004442666471004486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,768,0.004088533421357473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,1536,0.023492266734441124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,1024,0.021282132466634116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,512,0.00377813329299291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,768,0.019115734100341796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,512,0.017492266496022542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,256,0.0033333333830038703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,128,0.0030901332696278887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,256,0.016155733664830526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,64,0.0029056000212828318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,128,0.015783466895421348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,64,32,0.0029279999434947968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,64,0.015793066223462424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,64,32,0.014912000298500061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,65536,0.13433067003885907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,51200,0.10635626316070557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,16384,0.03682880004247029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,12288,0.029285333553949994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,16384,0.19610133171081542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,12288,0.15009493827819825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,10240,0.025860265890757246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,8192,0.020362667242685952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,10240,0.12679680188496906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,51200,0.5821386973063152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,7168,0.01432213286558787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,8192,0.10367146333058674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,65536,0.7382954915364583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,5120,0.009268266956011455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,7168,0.09117973645528157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,6144,0.07722986539204915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,4096,0.007854933540026348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,5120,0.0635253349939982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,3584,0.007518933216730754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,4096,0.04807360172271728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,3072,0.007083733379840851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,2560,0.006359466910362243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,3072,0.03472426732381185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,2048,0.00554666668176651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,2560,0.02991466720898946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,1536,0.00489386667807897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,2048,0.026891734202702838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,1024,0.004189866781234741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,1536,0.023822933435440063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,768,0.0038463999827702843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,1024,0.020606933037439983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,512,0.0034976000587145484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,768,0.018769067525863648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,256,0.0030965333183606463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,512,0.01771093408266703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,128,0.0029056000212828318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,256,0.016241066654523215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,64,0.00271573339899381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,128,0.01566506624221802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,32,0.00278613343834877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,64,0.01492586632569631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,32,0.014893866578737893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,512,0.008102400104204814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,256,0.006809600194295247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2816,32,6144,0.010284800330797832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,128,0.005611733098824819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2816,128,768,0.009731200337409974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2816,32,3584,0.040453334649403885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,16384,4.162071482340495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,12288,3.1046101888020834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,16384,5.465743001302084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,16384,7.363350423177083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,12288,5.510695393880209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,12288,3.978362782796224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,10240,2.534262339274089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,8192,2.0629397074381512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,10240,3.3194559733072913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,8192,3.740275319417318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,10240,4.686883036295573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,8192,2.5905227661132812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,51200,12.648291015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,7168,2.1107818603515627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,7168,2.454781850179036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,6144,1.5563947041829427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,6144,2.8400596618652343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,7168,3.97728525797526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,6144,2.2333193461100262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,5120,1.4748767852783202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,5120,2.659754689534505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,5120,1.961629867553711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,4096,1.024011739095052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,4096,1.9654324849446614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,3584,0.9207914352416993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,3072,0.9554549535115561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,3584,1.214788309733073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,3584,1.794551467895508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,3072,1.5307487487792968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,3072,0.9884554545084635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,2560,0.6837418874104817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,2560,0.8346090952555338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,2048,0.9369386672973633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,2048,0.5344128290812175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,2560,1.445242691040039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,2048,0.7958016077677409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,1536,0.4403264045715332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,1536,0.6889151891072591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,1536,0.606170654296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,1024,0.30486825307210286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,1024,0.5597045262654622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,768,0.39840854008992516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,768,0.2560330708821615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,512,0.30801385243733725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,512,0.2009333292643229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,4096,1.2873621622721354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,512,0.3284576098124186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,256,0.19191039403279622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,256,0.1651306629180908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,128,0.15132800738016766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,128,0.1549525260925293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,256,0.31264959971110023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,64,0.14031893412272137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,128,0.24387200673421225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,64,0.15088426272074382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,32,0.13705280621846516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,65536,32,0.14657066663106283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,768,0.3949055989583333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,51200,23.234326171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,65536,51200,24.14927978515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,65536,1024,0.4142175992329915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,16384,3.350587717692057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,16384,6.626088460286458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,16384,4.162711588541667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,51200,9.844503784179688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,12288,2.4921590169270833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,12288,4.384528096516927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,65536,13.88072509765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,12288,4.02394765218099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,10240,1.953641637166341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,10240,4.305046590169271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,10240,2.816054280598958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,8192,1.663867696126302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,8192,3.0848724365234377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,7168,1.4438592274983724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,51200,19.018035888671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,8192,2.0723690032958983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,51200,17.44148152669271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,7168,2.688196309407552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,7168,1.774341328938802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,6144,1.184045918782552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,6144,1.5246859232584635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,6144,2.3589024861653645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,5120,1.9326741536458332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,5120,1.2599700927734374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,4096,0.7875839869181316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,4096,1.540497080485026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,3584,0.7175530751546224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,4096,1.0956916809082031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,5120,1.012897046407064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,3584,1.3681908925374349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,3584,0.9286325454711915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,3072,0.6312693277994792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,3072,1.1466784159342447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,2560,0.5774272282918294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,3072,0.8226986567179362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,2560,0.8982655843098959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,2560,0.6924906412760417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,2048,0.4723541259765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,65536,24.725044759114585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,65536,24.00383097330729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,1536,0.34388907750447595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,2048,0.6138026555379231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,1536,0.6154901504516601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,1536,0.5373994827270507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,1024,0.3830058733622233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,768,0.22742187182108559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,1024,0.349783452351888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,768,0.30115092595418297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,1024,0.2616213321685791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,256,0.14010987281799317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,512,0.23234027226765952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,512,0.16514986356099445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,768,0.34770027796427405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,256,0.15266985893249513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,128,0.12179733117421468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,256,0.22127040227254233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,128,0.1253397305806478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,128,0.18924694061279296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,64,0.12666133244832356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,32,0.10925227006276447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,64,0.12358506520589192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,51200,32,0.12015573183695476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,51200,2048,0.7854496002197265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,51200,512,0.25339412689208984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,16384,1.0674485524495443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,16384,1.8961322784423829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,51200,3.6317428588867187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,16384,1.338699722290039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,12288,1.4261183420817056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,65536,4.595116678873698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,51200,4.731404622395834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,12288,0.8258431752522787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,12288,0.9657983779907227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,10240,0.6728170394897461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,10240,1.1100031534830728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,10240,0.8115050633748373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,8192,0.5783765157063802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,65536,6.06323496500651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,51200,6.165100606282552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,7168,0.5356693267822266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,7168,0.5723189036051433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,7168,0.8462495803833008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,6144,0.6642474492390951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,6144,0.43145065307617186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,6144,0.5021514574686686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,5120,0.37700586318969725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,5120,0.5806901295979817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,5120,0.4270784060160319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,8192,0.9771370569864908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,4096,0.4477663993835449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,4096,0.3043903986612956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,4096,0.3475520133972168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,65536,7.606329345703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,3584,0.4208064079284668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,3584,0.26593813896179197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,3072,0.23624107042948403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,3584,0.2980863889058431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,3072,0.36245225270589193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,3072,0.26515626907348633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,2560,0.21302080154418945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,2048,0.22970879872639977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,2560,0.31935466130574547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,2048,0.1582101345062256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,1536,0.17783253987630207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,2048,0.20740693410237632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,1536,0.14473493893941242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,1024,0.09669546286265054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,1024,0.13909759521484374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,1536,0.16304853757222493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,768,0.10646080176035563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,1024,0.127621332804362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,768,0.09176426728566488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,512,0.08199893633524577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,768,0.1038581371307373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,512,0.06536853313446045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,256,0.049202132225036624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,256,0.05638613303502401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,512,0.09416639804840088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,128,0.04010133345921834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,64,0.03723306655883789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,256,0.08701653480529785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,128,0.06949653625488281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,64,0.05234453280766806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,32,0.053115733464558924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,8192,0.6368309020996094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,16384,128,0.050614400704701745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,16384,32,0.037257599830627444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,16384,2560,0.2171658674875895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,16384,1.3857643127441406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,16384,0.8395061492919922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,65536,3.486608123779297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,16384,1.0416757583618164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,51200,2.7805419921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,12288,0.6430079778035481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,65536,4.373515828450521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,51200,4.418330891927083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,12288,1.0292181650797525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,10240,0.8914688110351563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,10240,0.5618922551472981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,10240,0.6090549468994141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,8192,0.6840138753255208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,8192,0.44639466603597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,8192,0.5025482813517252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,65536,6.126139831542969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,7168,0.400216547648112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,7168,0.6693834940592448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,7168,0.4522762616475423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,6144,0.37822186152140297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,6144,0.5379456202189128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,6144,0.3536426544189453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,5120,0.28310934702555335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,5120,0.5507541020711263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,4096,0.22834879557291665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,4096,0.2833888053894043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,4096,0.3723381360371908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,3584,0.3863552093505859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,3584,0.2460458596547445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,3072,0.27903572718302405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,3072,0.22962026596069335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,3072,0.21046719551086426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,2560,0.23650879859924318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,2560,0.17280106544494628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,2048,0.19453120231628418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,2560,0.1795530637105306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,2048,0.1328704039255778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,1536,0.13472639719645182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,2048,0.17439254124959308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,1536,0.10831039746602375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,1536,0.12182719707489013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,51200,3.1446367899576826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,1024,0.08387200037638345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,1024,0.09332373142242431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,12288,0.7317333221435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,768,0.07094506422678629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,768,0.1001856009165446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,768,0.08210773468017578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,512,0.0648629347483317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,256,0.038007465998331706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,512,0.05775573253631592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,512,0.08759146531422933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,256,0.04828159809112549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,128,0.032737066348393754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,256,0.05680960019429525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,3584,0.20065706570943198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,64,0.029573333263397217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,128,0.04565226634343465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,128,0.06343146562576293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,64,0.043211734294891356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,32,0.0298634668191274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,12288,32,0.04365866581598918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,12288,1024,0.10668373107910156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,12288,5120,0.3126357396443685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,16384,1.1255498250325522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,16384,0.74246826171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,51200,2.3793846130371095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,16384,0.8328042984008789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,12288,0.8984863917032877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,51200,2.6243573506673177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,12288,0.573470942179362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,65536,3.189441172281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,12288,0.6359797159830729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,10240,0.4805472056070964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,51200,3.7870699564615884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,10240,0.5176341374715169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,10240,0.7652437210083007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,8192,0.6252202351888021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,8192,0.4206581433614095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,7168,0.3641066551208496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,7168,0.5080949465433757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,6144,0.4655413309733073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,6144,0.3106176058451334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,6144,0.32127787272135416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,5120,0.2605621337890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,65536,4.866169738769531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,5120,0.37665065129597985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,5120,0.27348372141520183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,4096,0.296833070119222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,4096,0.20984427134195963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,3584,0.26335892677307127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,3584,0.19023253122965494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,3584,0.20710399945576988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,4096,0.24767786661783853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,3072,0.22365973790486654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,3072,0.16601600646972656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,3072,0.17120213508605958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,2560,0.1415669282277425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,2560,0.18994240760803222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,2048,0.15201706886291505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,2048,0.11592960357666016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,2560,0.15311573346455892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,1536,0.12187306880950928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,2048,0.1272981325785319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,1024,0.08411626815795899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,1536,0.09687893390655518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,1536,0.1013760010401408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,1024,0.08189333279927571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,1024,0.07553280194600423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,768,0.06754879951477051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,768,0.061001598834991455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,512,0.05021760066350302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,768,0.06818772951761881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,512,0.05057706832885742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,256,0.03389333486557007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,512,0.05790826479593912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,256,0.0412501335144043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,128,0.02759573260943095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,256,0.052817066510518394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,64,0.024997333685557045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,128,0.039046398798624676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,128,0.04438826640446981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,10240,32,0.024995199839274087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,64,0.0401802659034729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,10240,32,0.03970880111058553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,65536,3.6059679667154945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,7168,0.3592501322428385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,10240,8192,0.3993920008341471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,16384,0.9668469111124673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,65536,2.4778676350911457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,51200,2.0043275197347006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,51200,3.167948913574219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,16384,0.6616917292277018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,12288,0.7163082758585613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,12288,0.4786869366963704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,65536,4.1719006856282554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,10240,0.571234130859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,10240,0.42258345286051435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,8192,0.32719786961873376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,8192,0.5112213452657064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,7168,0.40087254842122394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,7168,0.3197696050008138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,6144,0.37732372283935545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,6144,0.26436053911844887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,12288,0.65480531056722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,5120,0.2934922536214193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,5120,0.21588160196940104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,16384,0.8960800170898438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,4096,0.23053654034932455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,4096,0.17516266504923503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,10240,0.5427008310953776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,6144,0.3238752047220866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,3584,0.15073173840840656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,3584,0.20290560722351075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,65536,4.189777119954427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,3072,0.17555413246154786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,3072,0.16244907379150392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,2560,0.15585813522338868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,7168,0.38446079889933266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,2560,0.12506026426951092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,2048,0.12882026831309001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,2048,0.10283946990966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,51200,3.008101399739583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,8192,0.4280266761779785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,1536,0.10946239630381267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,1536,0.08276693026224771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,1024,0.06861759821573893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,1024,0.06496853431065877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,3072,0.1605504035949707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,768,0.05322240193684896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,768,0.05422293345133463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,1536,0.08171306451161703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,512,0.04097599983215332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,512,0.044549334049224856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,4096,0.22247360547383627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,256,0.02841599980990092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,256,0.036908801396687826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,2560,0.1393770694732666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,128,0.022627200682957968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,128,0.03357013463973999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,5120,0.26879361470540364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,64,0.020114133755366005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,64,0.03419839938481649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,8192,32,0.020577067136764528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,8192,32,0.03385173479715983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,3584,0.18961386680603026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,1024,0.06583679914474487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,2048,0.11138239701588948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,65536,2.216286977132161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,65536,2.4497940063476564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,65536,3.287835693359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,51200,2.66856206258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,16384,0.7731007893880208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,16384,0.5821727752685547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,51200,1.7499530792236329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,128,0.028627200921376543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,256,0.03375786542892456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,51200,1.9131467183430988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,12288,0.4269664128621419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,12288,0.590937614440918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,16384,0.5785205205281575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,12288,0.42803306579589845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,10240,0.46935361226399736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,8192,0.3035818735758463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,512,0.04066346486409505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,8192,0.4389472007751465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,10240,0.4013674736022949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,8192,0.2952853202819824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,7168,0.26340907414754233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,7168,0.33239040374755857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,7168,0.25827946662902834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,6144,0.29944213231404626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,6144,0.2279797395070394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,6144,0.22930666605631508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,5120,0.2683786710103353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,5120,0.19565973281860352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,4096,0.19885120391845704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,8192,768,0.05054719845453898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,4096,0.1585792064666748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,4096,0.17279574076334636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,3584,0.17411519686381022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,3584,0.142794672648112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,3072,0.14963626861572266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,3072,0.13383679389953612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,2560,0.13076693216959637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,2560,0.10553173224131267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,2560,0.11860586802164715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,2048,0.10732373396555583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,2048,0.08919466336568196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,1536,0.08321173191070556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,2048,0.10398186842600506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,1024,0.06619733174641927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,1536,0.07721920013427734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,1024,0.05935573180516561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,768,0.04951999982198079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,768,0.05052479902903238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,768,0.051463464895884194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,512,0.037673600514729816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,5120,0.18306986490885419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,512,0.04248960018157959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,512,0.041630931695302326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,256,0.026312534014383954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,256,0.03601386547088623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,256,0.034140801429748534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,128,0.020300799608230592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,128,0.031724800666173295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,128,0.03325440088907878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,64,0.01774719953536987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,7168,32,0.01816213329633077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,3072,0.13041600386301677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,32,0.03142399986584981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,1536,0.07436373233795165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,1024,0.05954240163167318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,10240,0.35819946924845375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,7168,64,0.03146026730537414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,7168,3584,0.13910719553629558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,65536,1.9239200592041015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,16384,0.6330997467041015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,16384,0.4905205408732097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,51200,1.5327359517415364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,12288,0.3760608037312826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,65536,2.2005236307779947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,16384,0.5146378517150879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,51200,1.7259349822998047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,12288,0.5312255859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,51200,2.2039894104003905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,65536,2.859387715657552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,12288,0.400326410929362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,8192,0.28527466456095374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,10240,0.40093758900960286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,8192,0.2899829228719076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,10240,0.3658709208170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,8192,0.3900320053100586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,7168,0.24172587394714357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,7168,0.28509546915690104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,7168,0.24818347295125326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,6144,0.21112319628397622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,5120,0.2245994726816813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,5120,0.17162346839904785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,5120,0.17789759635925292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,4096,0.16967253684997557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,4096,0.1412992000579834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,3584,0.14716480573018392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,3584,0.1225162665049235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,4096,0.15129173596700032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,3072,0.12560959657033283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,3072,0.10850240389506023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,3072,0.1096725304921468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,2560,0.09498666922251384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,2560,0.09500586986541748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,2048,0.08635839621225992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,2048,0.08048213322957357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,2048,0.08022933006286621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,1536,0.06703573067982992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,1024,0.049046401182810465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,1536,0.06502613226572672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,1536,0.06608746846516927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,768,0.03987520138422648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,1024,0.05209920008977255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,768,0.04528106848398845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,768,0.04538666804631551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,512,0.03065706690152486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,512,0.03730133374532064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,512,0.038101331392923994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,256,0.020939733584721884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,256,0.030869332949320476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,256,0.03214079936345418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,128,0.016823466618855795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,6144,0.255623467763265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,128,0.02786986629168193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,6144,0.20894187291463218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,128,0.02999146580696106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,64,0.014282666643460593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,32,0.016084266702334087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,32,0.02927680015563965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,6144,2560,0.11349226633707683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,1024,0.05144000053405762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,10240,0.3239509264628092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,6144,64,0.029649066925048827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,6144,3584,0.12446826299031574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,65536,2.4850496927897137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,16384,0.5423413594563802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,51200,1.8927552541097004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,51200,1.3562975565592448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,65536,2.482781982421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,16384,0.4689685185750325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,12288,0.45732908248901366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,16384,0.5760096232096354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,12288,0.3526442527770996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,10240,0.344158935546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,65536,1.7281514485677083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,10240,0.3011338551839193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,8192,0.27374614079793297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,12288,0.4244864145914714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,51200,2.0339584350585938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,7168,0.24010559717814126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,8192,0.25373546282450354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,6144,0.23134934107462563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,6144,0.19543466567993165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,5120,0.16847786903381348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,4096,0.1488661289215088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,4096,0.13066879908243817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,7168,0.21392107009887695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,3584,0.1259498675664266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,5120,0.17936533292134602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,3584,0.10976533095041911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,3072,0.10962133407592774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,3072,0.09803199768066406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,10240,0.355622386932373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,2560,0.0926975965499878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,2560,0.08526826699574788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,8192,0.2770496050516764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,2560,0.09421866734822591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,2048,0.07570239702860514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,6144,0.20625920295715333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,2048,0.07279360294342041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,1536,0.05901333491007486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,1536,0.060543998082478845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,1024,0.042822400728861496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,5120,0.17664426167805988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,1024,0.04820906718571981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,768,0.0355402668317159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,768,0.04183573325475057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,4096,0.14579092661539714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,512,0.02746880054473877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,512,0.03385920127232869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,7168,0.24913600285847984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,256,0.018262400229771932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,256,0.029208532969156903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,256,0.02448426683743795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,128,0.013988266388575235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,3584,0.12603092988332112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,128,0.02792746623357137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,64,0.012686933080355326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,64,0.02808213432629903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,5120,32,0.01332373321056366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,3072,0.1069002628326416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,5120,32,0.027801599105199176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,2048,0.07501333554585775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,1536,0.05565439860026041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,65536,1.4957003275553384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,1024,0.044506665070851645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,768,0.03588800032933553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,65536,1.9697994232177733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,512,0.02872106631596883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,16384,0.4368618647257487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,16384,0.4209343910217285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,12288,0.33048534393310547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,51200,1.1776042938232423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,12288,0.3124426523844401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,51200,1.495843251546224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,5120,128,0.019900800784428914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,10240,0.2517909367879232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,10240,0.2792319933573405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,8192,0.22397333780924478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,8192,0.20358079274495444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,7168,0.19928000768025717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,7168,0.1787061373392741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,6144,0.1709269364674886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,6144,0.15336000124613444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,12288,0.35520960489908854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,5120,0.1436789353688558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,5120,0.13064746856689452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,16384,0.4647509256998698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,4096,0.11959573427836101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,6144,0.176474666595459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,4096,0.11369173526763916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,10240,0.2991605440775553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,3584,0.10160533587137859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,3584,0.0963807980219523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,65536,2.1271903991699217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,3072,0.09816106955210367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,3072,0.09176106452941894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,7168,0.2057151953379313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,2560,0.07707413037618002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,2560,0.0752021312713623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,51200,1.6431456247965497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,8192,0.2291605313618978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,2048,0.0694922685623169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,2048,0.06430613199869792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,1536,0.04871466557184855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,1536,0.05318933327992757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,1024,0.034663466612497966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,1024,0.04212373495101929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,3072,0.08871893088022867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,768,0.02810879945755005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,768,0.03584853410720825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,4096,0.12171626885732015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,512,0.0216757337252299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,1536,0.045665065447489425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,512,0.030694399277369184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,256,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,256,0.02595733404159546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,5120,0.14653439521789552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,128,0.011991467078526814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,128,0.024497065941492716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,2560,0.07569279670715331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,64,0.009496532877286275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,64,0.024834134181340537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,4096,32,0.012407466769218445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,4096,32,0.02521173357963562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,3584,0.10282666683197021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,2048,0.06018133163452148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,65536,1.369970194498698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,65536,1.6117610931396484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,1024,0.038846933841705324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,51200,1.2913717905680338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,51200,1.0946517944335938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,65536,1.7035733540852864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,16384,0.42320534388224285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,16384,0.36144641240437825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,12288,0.31567999521891277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,16384,0.4155551910400391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,10240,0.255511474609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,10240,0.24824105898539223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,10240,0.25101332664489745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,128,0.015774933497111003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,256,0.01885120073954264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,8192,0.1905845324198405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,8192,0.18446613947550455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,7168,0.17593919436136882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,12288,0.27267519632975257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,7168,0.1625343958536784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,6144,0.1442911942799886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,7168,0.18117334047953287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,5120,0.12521706422170004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,6144,0.13951679865519206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,6144,0.15526506106058757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,4096,0.09847359657287598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,5120,0.12476053237915039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,512,0.023374933004379272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,4096,0.0976960023244222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,3584,0.08485653400421142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,4096,0.10588266849517822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,3072,0.07554026444753012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,3584,0.08778346379597982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,3072,0.08056533336639404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,3072,0.07773119608561198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,2560,0.06383680105209351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,2560,0.06885440349578857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,4096,768,0.02910826603571574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,2048,0.05190933148066203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,51200,1.2203370412190755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,2048,0.05757759809494019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,1536,0.04588266611099243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,1536,0.04671253363291423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,1024,0.028600533803304035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,1024,0.03853013515472412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,12288,0.2994624137878418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,768,0.024498132864634196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,768,0.03312000036239624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,8192,0.1936255931854248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,512,0.01920426686604818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,512,0.0285589337348938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,512,0.022381865978240968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,256,0.013797332843144735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,256,0.024462932348251344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,5120,0.1260480006535848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,128,0.010819199681282043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,128,0.022795732816060385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,128,0.015592533349990844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,3584,0.09027413527170816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,64,0.00997440020243327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3584,32,0.01055999994277954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,64,0.02369813323020935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3584,32,0.023280000686645506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,2560,0.0672917366027832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,2048,0.05548266569773356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,1536,0.042822400728861496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,1024,0.034584534168243405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,65536,1.2947157541910808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,768,0.02799786726633708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,51200,1.0612021128336588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,65536,1.2639840443929038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,65536,1.157693862915039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,51200,0.9735904057820639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,16384,0.31328214009602867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,12288,0.2516416072845459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,16384,0.28857173919677737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,16384,0.36918185551961263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,12288,0.22899840672810873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,12288,0.2512928009033203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,10240,0.22564800580342612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,10240,0.2126527945200602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,8192,0.16495253245035807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,8192,0.16971200307210285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,51200,0.9268853505452475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,7168,0.14108907381693522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,8192,0.15922346115112304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,7168,0.15631359418233234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,6144,0.13588266372680663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,7168,0.1336703936258952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,6144,0.129693865776062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,5120,0.10270400047302246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,5120,0.09992746512095133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,5120,0.12090026537577311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,4096,0.08313173453013102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,4096,0.09476799964904785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,3584,0.0780512015024821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,3584,0.08125973542531331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3584,256,0.018773333231608073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,3584,0.0757269303003947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,2560,0.05565439860026041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,3072,0.07190506458282471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,2560,0.05500586827596029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,2560,0.06521919965744019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,2048,0.04381759961446126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,2048,0.053964801629384364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,2048,0.04753066698710124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,1536,0.035069866975148516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,1536,0.04224533240000407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,1024,0.024497065941492716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,1024,0.030763733386993408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,768,0.020233599344889323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,768,0.029410133759180706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,6144,0.12277546723683674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,512,0.015425067146619162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,512,0.024579199155171712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,3072,0.0679754654566447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,256,0.01069546639919281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,256,0.021848533550898233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,256,0.018666666746139527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,128,0.00842026670773824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,128,0.021623466412226358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,128,0.01704853375752767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,64,0.008293333152929943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,1536,0.039418665568033855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,64,0.02121280034383138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,3072,32,0.00867306689421336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,32,0.02217493255933126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,3072,1024,0.03338346481323242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,10240,0.1829919974009196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,65536,1.0997428894042969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,4096,0.08194560209910075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,65536,1.1447797139485678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,3072,0.06387733221054077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,51200,0.8956799825032553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,16384,0.26417600313822426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,16384,0.30096747080485026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,12288,0.19968852996826172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,12288,0.22992426554361978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,51200,0.9231701532999674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,10240,0.1721109390258789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,768,0.02678080002466838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,10240,0.19750399589538575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,8192,0.13514773050944012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,8192,0.15691733360290527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,3072,512,0.022050132354100548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,7168,0.11853973070780437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,7168,0.13605012893676757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,6144,0.10354986985524495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,6144,0.11804052988688152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,12288,0.2563349405924479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,5120,0.08553067048390707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,5120,0.10009706815083821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,65536,1.38066775004069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,4096,0.07376213073730468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,16384,0.3353674570719401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,4096,0.0904298702875773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,6144,0.12675519784291583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,3584,0.06507413387298584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,3584,0.07460266749064128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,3072,0.05383040110270182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,3072,0.06574399868647257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,10240,0.21241599718729653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,2560,0.04631466468175252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,7168,0.15228479703267414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,2560,0.05718613465627035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,51200,1.0950026194254556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,2048,0.04084800084431966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,2048,0.05122559865315755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,8192,0.1713215986887614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,1536,0.02858346700668335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,1536,0.03823786576588949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,1024,0.02109653353691101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,1024,0.03058133323987325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,3072,0.06576000054677328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,768,0.01716266671816508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,768,0.0264789342880249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,4096,0.08833706378936768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,512,0.01332586705684662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,1536,0.034620801607767746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,512,0.024899200598398844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,256,0.009354666868845622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,256,0.020730666319529214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,2560,0.0567413330078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,128,0.007613866527875264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,128,0.01952426632245382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,5120,0.10958080291748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,64,0.006783999999364217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,64,0.019402666886647543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,3584,0.07484587033589682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2560,32,0.0071381335457166035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2560,32,0.019419733683268228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,2048,0.045425065358479816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,65536,1.2501642862955729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,65536,1.2219605763753256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,51200,0.9812992095947266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,51200,0.9525802612304688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,16384,0.2993429183959961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,1024,0.02704213261604309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,16384,0.3177290598551432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,16384,0.2574378649393717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,12288,0.2198570728302002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,12288,0.24133973121643065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,12288,0.20047574043273925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,10240,0.18602026303609212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,10240,0.20514453252156578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,10240,0.1722559928894043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,8192,0.14848747253417968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,8192,0.16520427068074545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,7168,0.13038933277130127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,7168,0.14548586209615072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,256,0.014621866742769876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,7168,0.12550933361053468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,6144,0.11332053343454998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,6144,0.12521920204162598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,128,0.012640000383059184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,5120,0.09548906485239664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,5120,0.10532053311665852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,5120,0.0847541332244873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,4096,0.07738026777903238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,4096,0.08589226404825846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,65536,1.0435424168904623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,3584,0.06947946548461914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,512,0.017646932601928712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,3584,0.08284693559010824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2560,768,0.021576533714930214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,3072,0.059572267532348636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,3072,0.0703648010889689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,2560,0.04876266717910767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,2560,0.05948906739552816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,2048,0.039162667592366536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,2048,0.0479637344678243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,2048,0.04046186606089274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,1536,0.030220800638198854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,1536,0.03849493265151978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,1536,0.03036160071690877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,1024,0.021522132555643718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,51200,0.8364320119222006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,1024,0.030542933940887453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,768,0.016784000396728515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,768,0.025939200321833295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,768,0.019703465700149535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,512,0.012583466370900473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,512,0.02328746716181437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,8192,0.13448853492736818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,256,0.00890880028406779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,256,0.020282665888468422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,6144,0.10377493699391682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,128,0.006674133241176605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,128,0.019220266739527384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,128,0.011190399527549744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,4096,0.06994453271230062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,64,0.006117333471775055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,2048,32,0.006421333551406861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,64,0.019036799669265747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,2048,32,0.01895573337872823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,3584,0.06212586561838786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,3072,0.05262826681137085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,2560,0.04772160053253174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,65536,0.619163703918457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,51200,0.4903626759847005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,65536,0.7196352005004882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,16384,0.16041599909464518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,65536,0.9296672185262045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,1024,0.02369813323020935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,51200,0.7253952026367188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,16384,0.258351993560791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,12288,0.11765440305074056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,10240,0.09881599744160971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,12288,0.18482666015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,512,0.015737600127855935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,10240,0.12033706506093342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,10240,0.1575061321258545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,8192,0.08061546484629313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,8192,0.09737813472747803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,8192,0.1282528003056844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,7168,0.07105813026428223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,6144,0.06127680142720541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,7168,0.08603200117746988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,7168,0.1144543965657552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,5120,0.05126293500264486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,5120,0.08443520069122315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,2048,256,0.01320746640364329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,4096,0.042498131593068436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,4096,0.06786666711171468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,6144,0.09885866641998291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,3584,0.037249068419138595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,3584,0.060311468442281094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,51200,0.5383199691772461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,3072,0.03259200056393941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,16384,0.18024959564208984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,3072,0.0523199995358785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,3072,0.041621331373850504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,2560,0.02759679953257243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,2560,0.04416106541951497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,2048,0.022921599944432578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,2560,0.03548906644185384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,2048,0.037587201595306395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,1536,0.031215999523798627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,6144,0.07479893366495768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,1024,0.013614933689435324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,1024,0.025889066855112712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,1024,0.01935466726620992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,768,0.011432533462842304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,768,0.023893332481384276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,768,0.017384533087412515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,512,0.009111467003822326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,512,0.021048533916473388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,4096,0.05185706615447998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,256,0.006903466582298279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,12288,0.14359572728474934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,256,0.019237333536148073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,1536,0.018410666783650716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,256,0.012388267119725545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,128,0.017923200130462648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,128,0.010969600081443787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,64,0.005323733389377594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,32,0.005494399865468343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,64,0.017499732971191406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1536,32,0.01773759921391805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,65536,0.45542720158894856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,5120,0.06320000092188517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,3584,0.04638719956080119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,51200,0.3487680117289225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,65536,0.8416437149047852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,16384,0.11538453102111816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1536,128,0.0058773333827654515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,51200,0.65730775197347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,16384,0.2184373378753662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,12288,0.08902186552683512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,12288,0.167796262105306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,10240,0.07183252970377604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,10240,0.14380906422932943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,10240,0.12572480042775472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,8192,0.05800426801045736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,8192,0.11762773195902507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,512,0.014659200112024942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,7168,0.05284266471862793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,2048,0.029820799827575684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,7168,0.10365119775136311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,6144,0.04379520018895467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,6144,0.08993813196818033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,65536,0.7630805333455404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1536,1536,0.02468693256378174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,5120,0.037979733943939206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,51200,0.5906293233235677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,16384,0.19619520505269367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,5120,0.0657525340716044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,5120,0.07579306761423746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,4096,0.031030400594075518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,4096,0.059164798259735106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,3584,0.027553067604700728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,3072,0.02322453260421753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,12288,0.14848960240681966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,3584,0.052426668008168545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,2560,0.019795199235280357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,3072,0.04485013484954834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,2560,0.03864959875742595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,2560,0.033454934755961105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,2048,0.016787199179331462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,2048,0.03322453300158183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,1536,0.013656533757845559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,1536,0.028566400210062664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,8192,0.10015040238698322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,7168,0.08965013027191163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,1024,0.010289067029953003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,1024,0.023605332771937052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,6144,0.07518186569213867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,768,0.008703999718030294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,768,0.021651200453440347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,768,0.014129066467285156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,512,0.007177599767843883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,512,0.01984106699625651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,4096,0.053649067878723145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,256,0.005880533158779145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,256,0.018227199713389076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,256,0.009596799810727436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,3584,0.04634346564610799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,128,0.005029333134492239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,3072,0.03900800148646037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,64,0.00445119986931483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,128,0.017461333672205606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,64,0.017164800564448038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,2048,0.027723733584086103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,1024,32,0.004917333523432413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,1024,32,0.017407999436060587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,1536,0.02145493427912394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,65536,0.3407338778177897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,51200,0.27103894551595054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,65536,0.7862666447957356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,51200,0.37979199091593424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,16384,0.08860692977905274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,51200,0.6287562688191731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,12288,0.0673408031463623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,16384,0.2059669335683187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,12288,0.15981760025024414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,10240,0.05559146801630656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,10240,0.08075839678446452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,10240,0.13665919303894042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,8192,0.0452672004699707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,1024,0.016561067104339598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,8192,0.06572800079981486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,7168,0.04017066558202108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,8192,0.11230186621348064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,6144,0.03450453281402588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,7168,0.09873279730478922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,6144,0.08583040237426758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,5120,0.03219946622848511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,512,0.011428266763687134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,5120,0.07168746789296468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,4096,0.025156267484029132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,4096,0.05613546768824259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,1024,128,0.008182399968306223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,3584,0.0214901328086853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,3584,0.04861439863840739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,3584,0.03293866713841756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,3072,0.01895893414815267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,3072,0.041902931531270345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,65536,0.48061440785725906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,2560,0.016807466745376587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,2560,0.03649386564890544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,16384,0.1271018664042155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,2048,0.01360106666882833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,2048,0.03099520007769267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,12288,0.09700160026550293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,2048,0.020959999163945517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,1536,0.01102186640103658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,1536,0.026477867364883424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,1024,0.008593066533406576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,1024,0.023670399188995363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,1024,0.01414293348789215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,768,0.00746666689713796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,6144,0.05125759840011597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,7168,0.058413867155710855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,768,0.02101226647694906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,768,0.012587733070055642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,512,0.0062165334820747375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,256,0.005107200145721436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,256,0.017821866273880004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,5120,0.04347519874572754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,128,0.004794666667779287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,4096,0.03677440087000529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,128,0.01662826637427012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,64,0.004046933352947235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,64,0.01630400021870931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,768,32,0.004073599974314371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,32,0.0163263996442159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,65536,0.23661227226257325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,768,512,0.0191210667292277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,65536,0.7433909098307292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,3072,0.028658133745193482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,51200,0.18813652992248536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,2560,0.02485760052998861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,51200,0.6156063715616862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,16384,0.08276693026224771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,16384,0.19932053883870443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,1536,0.01760106682777405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,12288,0.06665706634521484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,12288,0.1555722713470459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,512,0.010866133371988933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,10240,0.056782933076222744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,10240,0.13254933357238768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,8192,0.04226773182551066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,256,0.009100799759229023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,768,128,0.008134399851163227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,8192,0.10960533618927001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,7168,0.040881065527598064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,7168,0.09505919615427652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,65536,0.5325951894124349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,6144,0.03210560083389282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,6144,0.08216319878896078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,51200,0.41689386367797854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,5120,0.02789013385772705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,5120,0.06809386412302652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,16384,0.1446986675262451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,4096,0.019885865847269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,4096,0.05150933265686035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,4096,0.04025493462880452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,3584,0.01969919999440511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,3584,0.044088534514109295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,10240,0.08934826850891113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,3072,0.016823466618855795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,3072,0.038396799564361574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,8192,0.07428373495737711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,2560,0.013843199610710144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,7168,0.06449600060780844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,2560,0.033446399370829265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,2048,0.011496532956759136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,2048,0.029345067342122395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,1536,0.009981866677602131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,1536,0.025227733453114826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,12288,0.10683306852976482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,1024,0.007206400235493978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,1024,0.02183253367741903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,6144,0.05556053320566813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,768,0.00626453310251236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,768,0.02053546706835429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,768,0.011211733023325603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,512,0.005134933193524678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,5120,0.04787520170211792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,512,0.019241599241892497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,256,0.0044266665975252785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,256,0.017280000448226928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,3584,0.03485226631164551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,128,0.0038954667747020722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,3072,0.03015786608060201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,128,0.01613866686820984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,64,0.003576533248027166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,64,0.01606613298257192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,512,32,0.003833599885304769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,2560,0.02524159948031108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,512,32,0.016076800227165223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,65536,0.14402666091918945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,2048,0.021437867482503255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,51200,0.11591040293375651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,1536,0.016729599237442015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,65536,0.6925493240356445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,1024,0.0132341335217158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,16384,0.04540266593297322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,51200,0.5477290471394857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,12288,0.03601386547088623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,16384,0.18754453659057618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,12288,0.14468587239583333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,10240,0.032202666997909545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,10240,0.12267413139343261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,512,0.009226666887601216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,8192,0.024804266293843587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,256,0.007672533392906189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,8192,0.09981439908345541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,7168,0.02109653353691101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,7168,0.08785066604614258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,512,128,0.006428800026575724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,6144,0.016961065928141277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,6144,0.07564906279246011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,65536,0.49087146123250325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,5120,0.014613333344459533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,5120,0.0628383994102478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,5120,0.044012800852457686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,4096,0.01225386659304301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,51200,0.3840277353922526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,16384,0.13063680330912272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,3584,0.011341866850852967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,12288,0.09823253154754638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,3584,0.04012159903844197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,3072,0.010215466221173603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,3072,0.03612906535466512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,10240,0.08220907052357992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,2560,0.010451199611028035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,2560,0.03099946578343709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,8192,0.06790826320648194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,4096,0.04779733419418335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,2048,0.008930133779843648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,2048,0.027347199122111004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,1536,0.007899733384450276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,7168,0.059279998143514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,1536,0.024220800399780272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,1024,0.0058890665570894875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,1024,0.021717333793640138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,6144,0.05057599941889444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,768,0.005197866757710775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,768,0.01913706660270691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,768,0.01016960044701894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,512,0.004418133199214936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,512,0.018963199853897095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,4096,0.03612906535466512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,2048,0.019662932554880778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,3584,0.031573333342870075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,256,0.003772799919048945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,256,0.016801067193349204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,1536,0.015336533387502035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,128,0.003421866645415624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,1024,0.01183146635691325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,64,0.00329066663980484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,3072,0.027428267399470012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,256,32,0.003365333378314972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,64,0.015256533026695251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,32,0.015411200126012168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,2560,0.022875734170277915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,65536,0.1481493314107259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,51200,0.1058847983678182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,65536,0.6836896260579427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,51200,0.5389610926310222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,16384,0.04020373423894246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,256,128,0.015938133001327515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,16384,0.12134933471679688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,12288,0.03217493295669556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,12288,0.14100693066914877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,10240,0.02762986620267232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,10240,0.11919786930084228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,512,0.008154666423797608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,8192,0.019897600015004478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,16384,0.18149867057800292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,8192,0.09726719856262207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,7168,0.015438933173815408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,256,0.006729599833488464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,256,128,0.005796266595522562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,6144,0.013719466328620911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,7168,0.08512213230133056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,6144,0.07353386878967286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,5120,0.011332266529401143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,5120,0.05827200015385946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,65536,0.46373545328776045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,4096,0.009675733248392741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,4096,0.04439146518707275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,51200,0.37378454208374023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,3584,0.009970133503278095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,3584,0.038341331481933597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,12288,0.0929738680521647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,3072,0.008282666901747386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,3072,0.03391466538111369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,10240,0.0781984011332194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,2560,0.007301333546638489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,2560,0.028835199276606244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,8192,0.06354453166325888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,2048,0.006593066453933716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,2048,0.02609813412030538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,7168,0.05672426621119181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,2048,0.0184170663356781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,1536,0.006461866696675618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,6144,0.049148801962534586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,1536,0.02316373387972514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,1024,0.004962133367856344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,1024,0.020426666736602782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,768,0.004473599791526795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,5120,0.04189333518346151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,768,0.01916159987449646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,512,0.003929600119590759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,512,0.01742186745007833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,4096,0.034086398283640545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,256,0.0034634667138258614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,256,0.016167466839154564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,3584,0.030059732993443805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,128,0.0031744000812371576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,128,0.015532799561818442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,3072,0.025726934274037678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,64,0.0030602666238943735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,64,0.01563093364238739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,128,32,0.0030879999200503034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,128,32,0.01514346698919932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,2560,0.022101332743962608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,65536,0.12770453294118245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,51200,0.10150506496429443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,65536,0.6828618367513021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,51200,0.5341695785522461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,16384,0.03720426559448242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,1536,0.014837333559989929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,12288,0.028242133061091107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,1024,0.011422933141390482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,10240,0.025889066855112712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,12288,0.14005759557088215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,768,0.00978559950987498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,8192,0.02108373244603475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,16384,0.18098665873209635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,7168,0.016064000129699708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,6144,0.013957333564758301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,10240,0.11811839739481608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,5120,0.011967999736467998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,7168,0.08422613143920898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,512,0.007855999966462452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,4096,0.010506666700045268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,6144,0.07205973466237386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,3584,0.009205333391825358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,5120,0.057615999380747476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,4096,0.04341333309809367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,3072,0.008603733777999879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,3584,0.03737066586812337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,2560,0.007665066421031952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,3072,0.03320853312810262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,2048,0.007002666592597961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,1536,0.005504000186920166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,2560,0.02940479914347331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,2048,0.026172800858815508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,1024,0.004500266909599304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,1536,0.023176532983779908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,1024,0.020243199666341145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,512,0.0036170666416486105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,768,0.018572799364725747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,256,0.003159466634194056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,512,0.017614932854970296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,128,0.002974933385848999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,256,0.01583999991416931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,64,0.002869333326816559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,128,0.015475199619928996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,32,0.0029120000700155893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,256,0.00658240020275116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,32,0.015011200308799743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,65536,0.12971413135528564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,51200,0.10378452936808269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,16384,0.03726293245951335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,16384,0.18047253290812176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,12288,0.028892799218495684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,65536,0.6814741134643555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,51200,0.534610112508138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,10240,0.024306132396062215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,12288,0.13897813161214193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,8192,0.019432532787322997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,7168,0.012777599692344665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,10240,0.11711253325144451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,6144,0.011481600006421407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,8192,0.09509013493855795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,7168,0.08300586541493735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,5120,0.010088533163070679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,4096,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,6144,0.07191572984059652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,3584,0.008370133241017659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,4096,0.04328639904658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,64,768,0.004149333387613296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,3584,0.036497068405151364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,2560,0.0072629332542419435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,3072,0.0326527992884318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,2048,0.006198399762312571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,2560,0.02884693344434102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,1536,0.005190399785836538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,64,0.015375999609629312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,2048,0.02613760034243266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,1024,0.004413866500059763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,1536,0.023204267024993896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,768,0.00402453343073527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,1024,0.020082134008407592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,512,0.0035584000249703727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,768,0.018538665771484376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,256,0.0031295999884605407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,512,0.01813546617825826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,128,0.0029418667157491045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,256,0.01581013302008311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,64,0.0028160000840822858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,128,0.015770666797955833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,32,0.0028042666614055633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,64,0.014775466918945313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,32,0.015095466375350952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,64,8192,0.09703466892242432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2561,128,128,0.005493333439032236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2561,32,5120,0.05694719950358072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2561,32,3072,0.007401599983374278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,12288,2.9326197306315107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,16384,4.069554138183594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,12288,5.575929768880209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,16384,7.320069376627605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,10240,2.560649617513021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,10240,4.611420694986979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,10240,3.15795415242513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,8192,1.9405301411946614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,51200,12.41536865234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,8192,3.730036163330078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,8192,2.9840660095214844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,16384,4.878611246744791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,7168,3.939003753662109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,7168,1.7980959574381512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,12288,3.7382527669270837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,6144,1.470919418334961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,6144,3.0475008646647135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,6144,1.8619626363118489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,5120,2.388720957438151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,5120,1.2009204864501952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,51200,18.491011555989584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,5120,1.6171051025390626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,4096,1.0375808080037436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,4096,1.8892779032389324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,3584,0.8631711959838867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,3584,1.0832010904947915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,3584,1.709008026123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,3072,0.7485013326009115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,3072,1.4750293731689452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,2560,0.6495370864868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,3072,0.9947114944458008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,2560,1.1523957570393881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,2048,0.5502400080362956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,2048,0.9359093348185221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,2048,0.7449973424275715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,1536,0.41248213450113935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,1536,0.7264810562133789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,1024,0.4893365224202474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,1024,0.3361248016357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,768,0.37281920115152994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,768,0.2547423998514811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,51200,24.19517618815104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,768,0.3800607999165853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,7168,2.2930068969726562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,1536,0.5660575866699219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,512,0.18900160789489745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,256,0.15634667078653972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,512,0.29205118815104164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,256,0.15805439949035643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,512,0.35590400695800783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,128,0.1482037385304769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,128,0.16399146715799967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,256,0.2451530615488688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,1024,0.45567998886108396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,64,0.13273173173268635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,128,0.23520426750183104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,65536,32,0.1464032014211019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,32,0.14109652837117512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,65536,64,0.14505279858907064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,4096,1.245031483968099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,65536,2560,0.7643103917439779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,16384,3.356949361165365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,16384,5.781813557942709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,51200,9.716311645507812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,16384,3.7344459533691405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,12288,2.8603530883789063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,65536,14.145390828450521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,12288,4.365848286946615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,12288,2.940226236979167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,51200,14.687579345703124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,10240,2.1000181833902998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,10240,3.8612246195475257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,10240,3.203959401448568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,8192,1.711517842610677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,8192,3.579071044921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,8192,1.9510837554931642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,7168,1.450505574544271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,51200,19.477278645833334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,65536,19.366861979166664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,7168,1.6930474599202472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,7168,2.5687103271484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,6144,1.264355214436849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,6144,2.353903961181641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,6144,1.4726730346679688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,5120,1.0148821512858073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,5120,1.2052885691324868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,4096,0.8682730356852213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,4096,1.4526762644449869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,4096,0.9935626983642578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,5120,1.8735722859700519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,3584,0.7725791931152344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,3584,1.3767008463541666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,3584,0.8855242411295572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,3072,0.6335818608601888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,3072,0.7489791870117187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,2560,0.5543488184611003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,3072,1.2199989318847657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,2560,0.9543978373209635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,2048,0.43119252522786455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,2560,0.6561109542846679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,2048,0.7642570495605469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,1536,0.39051198959350586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,2048,0.571946652730306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,1024,0.39094292322794594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,1536,0.5923381169637044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,1536,0.47459732691446943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,1024,0.2653792063395182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,768,0.22145919799804686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,768,0.32028481165568035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,1024,0.3704085350036621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,512,0.17907840410868328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,512,0.253654400507609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,256,0.14761279424031576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,768,0.3263061205546061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,128,0.11550186475118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,512,0.24543892542521156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,256,0.13711573282877604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,64,0.10601706504821777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,128,0.12416746616363525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,128,0.19697813987731932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,32,0.10322666962941487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,64,0.12000426451365154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,51200,32,0.11785600185394288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,51200,65536,24.79555867513021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,16384,1.927175521850586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,51200,256,0.19410667419433594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,51200,3.521382395426432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,16384,1.1867477416992187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,51200,3.9736063639322916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,16384,1.2409727732340494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,65536,4.505533854166666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,12288,0.9846474965413412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,12288,1.3932735443115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,10240,0.7707583745320638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,51200,5.769971211751303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,10240,1.1885578155517578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,10240,0.771677843729655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,8192,0.5753226598103841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,8192,1.012977091471354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,7168,0.8403647740681967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,7168,0.5424394607543945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,7168,0.5906858444213867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,6144,0.7258431752522786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,65536,8.011711120605469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,6144,0.4653941472371419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,6144,0.5144447962443034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,5120,0.33993705113728845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,8192,0.6317482630411784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,5120,0.6240896224975586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,5120,0.40563414891560867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,4096,0.4915050824483235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,12288,0.9152885437011719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,4096,0.29657707214355467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,4096,0.3099029223124186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,65536,5.339744059244792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,3584,0.39108158747355143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,3584,0.30324907302856446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,3584,0.2787957191467285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,3072,0.26645973523457844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,3072,0.37884480158487954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,3072,0.2520256042480469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,2560,0.2828277269999186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,2560,0.21476052602132162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,2560,0.22087146441141764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,2048,0.2648938655853271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,1536,0.1358282725016276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,1536,0.21343572934468588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,1536,0.15722559293111166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,1024,0.1270485321680705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,2048,0.21249173482259115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,1024,0.10662506421407063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,768,0.10445333321889241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,512,0.07406826814015707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,1024,0.1377237319946289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,768,0.10807573000590007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,512,0.07173546950022379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,512,0.09089492956797282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,256,0.04659413496653239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,128,0.04025706847508748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,256,0.07807679971059164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,256,0.057550934950510654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,128,0.05253333250681559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,16384,128,0.0705087979634603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,32,0.037512532869974774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,32,0.05278506676355997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,2048,0.1628149350484212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,768,0.09475306669871011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,16384,64,0.036210131645202634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,16384,64,0.052604798475901285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,16384,0.8295914967854818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,16384,1.372719955444336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,16384,0.9227157592773437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,65536,3.4826688130696617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,51200,2.690258026123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,12288,1.0885716756184896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,12288,0.6895637512207031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,65536,4.222706095377604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,51200,2.9805013020833333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,12288,0.6856704076131185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,10240,0.5458698908487956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,51200,4.488020324707032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,10240,0.8922250747680665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,10240,0.5899178822835286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,8192,0.5029888153076172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,8192,0.4798975944519043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,7168,0.41390612920125325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,7168,0.5993621190388997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,7168,0.423796272277832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,6144,0.551425043741862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,6144,0.3544074694315592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,6144,0.35371414820353186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,5120,0.30753920873006185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,5120,0.45427732467651366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,65536,5.607470703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,4096,0.24257599512736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,4096,0.252127997080485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,3584,0.3142560005187988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,3584,0.2147530714670817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,3584,0.2302453358968099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,3072,0.18758400281270343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,3072,0.2580298741658529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,3072,0.20917654037475586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,2560,0.16664212544759113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,2560,0.15845120747884114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,2048,0.1413866678873698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,1536,0.1368170738220215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,2048,0.14533653259277343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,8192,0.7444192250569661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,1536,0.10894186496734619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,1024,0.09537493387858073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,1536,0.11493333180745442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,1024,0.08001813093821207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,768,0.07702186902364096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,768,0.06898026466369629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,768,0.07703999678293863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,512,0.057316267490386964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,512,0.056859731674194336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,512,0.06575786670049032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,5120,0.32393067677815757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,256,0.039537068208058676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,128,0.031202133496602374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,256,0.05391573508580526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,4096,0.37293761571248374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,128,0.04223039944966634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,128,0.05012906789779663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,64,0.028917332490285236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,32,0.029721599817276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,64,0.04279786745707194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,32,0.043933868408203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,2560,0.21620160738627114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,12288,2048,0.18215893109639486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,12288,1024,0.0926197369893392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,12288,256,0.046748801072438555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,16384,1.169381332397461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,51200,2.307883707682292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,65536,3.0414100646972657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,16384,0.7606005350748698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,51200,2.5812586466471354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,65536,3.407690684000651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,16384,0.7983039855957031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,12288,0.8273248036702474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,51200,3.7461878458658853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,12288,0.5852170944213867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,10240,0.4984949429829915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,10240,0.5452437082926432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,10240,0.7207157135009765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,8192,0.5874314626057943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,8192,0.37672853469848633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,65536,4.5656895955403645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,7168,0.4868895848592122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,7168,0.36909545262654625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,7168,0.36762987772623695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,6144,0.44077545801798507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,6144,0.28811734517415366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,5120,0.24942612648010254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,5120,0.26110827128092445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,4096,0.20125865936279297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,4096,0.29131412506103516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,4096,0.21114560763041176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,3584,0.2514357407887777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,3584,0.20273173650105797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,3072,0.2121503988901774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,3072,0.1617429256439209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,3072,0.1738474686940511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,2560,0.181551996866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,2560,0.13797440528869628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,2560,0.15149973233540853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,2048,0.11453013420104981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,2048,0.12536319891611736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,1536,0.11189866860707601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,1536,0.09116480350494385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,1536,0.10114453633626301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,1024,0.07942293485005697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,1024,0.07081279754638672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,768,0.06345920165379842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,5120,0.36668478647867836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,768,0.06100053389867147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,768,0.06728106339772542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,3584,0.181712007522583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,512,0.05070933500925699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,512,0.05662293434143066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,256,0.033624533812204996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,256,0.04187306563059489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,256,0.04685866832733154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,12288,0.5705397288004558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,128,0.02786453366279602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,128,0.03769173224767049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,64,0.025098667542139692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,128,0.053863465785980225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,32,0.025994666417439777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,64,0.03824319839477539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,10240,32,0.040327465534210204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,2048,0.1545482635498047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,8192,0.39701118469238283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,10240,512,0.04800320068995158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,6144,0.2932703971862793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,16384,0.8835306803385417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,10240,1024,0.07648106416066489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,51200,1.9824063618977863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,65536,2.4664543151855467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,16384,0.6504416147867839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,65536,2.672778574625651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,51200,3.0768501281738283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,12288,0.4694410641988118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,12288,0.4889354705810547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,16384,0.6032533645629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,12288,0.6791231791178386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,10240,0.6177834828694662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,8192,0.31497812271118164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,10240,0.3987349192301432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,8192,0.45794986089070633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,65536,3.6737632751464844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,8192,0.35648959477742515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,10240,0.41913706461588546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,7168,0.42424427668253584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,7168,0.28178879419962566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,6144,0.243723726272583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,6144,0.27943572998046873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,6144,0.3352640151977539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,5120,0.20631999969482423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,5120,0.2962239901224772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,5120,0.20429333051045737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,4096,0.2353877385457357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,4096,0.17403732935587565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,3584,0.209879461924235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,3584,0.15555626551310223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,3584,0.1623957316080729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,3072,0.13709227244059247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,2560,0.14835947354634602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,2560,0.11691413720448811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,2560,0.12001279989878337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,2048,0.11678400039672851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,2048,0.10202986399332684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,2048,0.09603093465169271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,1536,0.09454399744669596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,1536,0.07960426807403564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,1536,0.08057920138041177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,1024,0.06483840147654216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,1024,0.06392960151036581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,7168,0.30338134765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,768,0.05298346678415934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,768,0.053858133157094326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,512,0.04036373297373454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,512,0.04552106857299805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,4096,0.16916799545288086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,256,0.02837120095888774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,512,0.04565653403600057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,256,0.03832426468531291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,256,0.03652799924214681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,3072,0.17117973963419597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,128,0.022001065810521445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,64,0.019661867618560792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,128,0.03327680031458537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,128,0.035835735003153485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,51200,2.0725888570149738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,8192,32,0.020716800292332967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,64,0.03392426570256551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,8192,32,0.03430399894714355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,3072,0.1285312016805013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,65536,2.296439361572266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,65536,2.3847989400227863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,1024,0.06181653340657552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,8192,768,0.0544053316116333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,51200,2.8181129455566407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,51200,1.78330561319987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,16384,0.5468586603800456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,16384,0.603114636739095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,12288,0.439464537302653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,16384,0.8402602513631185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,12288,0.6036095937093099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,12288,0.4647189458211263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,65536,3.422881062825521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,51200,1.8423914591471355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,10240,0.3781599998474121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,10240,0.5400970458984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,8192,0.28759787877400717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,8192,0.29382187525431314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,8192,0.43218453725179035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,7168,0.29117120107014977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,7168,0.37873493830362953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,6144,0.23694933255513512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,6144,0.22470720609029135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,6144,0.30965547561645507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,5120,0.2578282674153646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,5120,0.1998026688893636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,4096,0.15130772590637206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,4096,0.199509334564209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,3584,0.18320852915445965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,3584,0.135262934366862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,3584,0.13969173431396484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,3072,0.1503946622212728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,3072,0.12582826614379883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,3072,0.12137706279754638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,2560,0.12161813577016194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,2560,0.10808213551839192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,2560,0.1145290692647298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,2048,0.09917120138804117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,2048,0.0827392021814982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,2048,0.10404693285624186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,1536,0.0794111967086792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,1536,0.06813013553619385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,1024,0.0538154681523641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,1536,0.07354559898376464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,1024,0.053028265635172524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,768,0.042787198225657144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,768,0.0443125327428182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,512,0.03188160061836243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,768,0.05442879994710287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,256,0.022541866699854533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,512,0.038176000118255615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,512,0.045286401112874346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,10240,0.3656469345092773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,256,0.03332053422927857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,128,0.01831573247909546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,128,0.03166399995485942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,64,0.017820799350738527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,64,0.031506133079528806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,7168,32,0.018594133853912353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,7168,32,0.03191360036532084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,5120,0.20163307189941407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,7168,0.24674666722615562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,4096,0.17494719823201496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,65536,1.9884682973225911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,1024,0.05771733522415161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,256,0.03554240067799886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,7168,128,0.03302826682726542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,65536,3.0098602294921877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,16384,0.4998485247294108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,16384,0.6948256174723307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,12288,0.5342506408691406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,51200,2.3536351521809893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,51200,1.5794282277425131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,12288,0.3919829368591309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,10240,0.43817494710286453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,8192,0.3416895866394043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,10240,0.3584831873575846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,8192,0.28957014083862304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,7168,0.3058570543924967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,7168,0.24576746622721352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,6144,0.26101439793904624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,6144,0.21489599545796714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,16384,0.5589855829874675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,5120,0.2206432024637858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,12288,0.42511787414550783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,65536,2.4377451578776044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,5120,0.18266560236612955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,4096,0.14888745943705242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,4096,0.19021652539571127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,3584,0.165011199315389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,10240,0.3760565439860026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,51200,1.969805908203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,3584,0.13656214078267415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,8192,0.28760318756103515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,3072,0.15370987256368002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,3072,0.1153450647989909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,2560,0.13043413162231446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,2560,0.10061013698577881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,2048,0.09245759646097819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,2048,0.08262613614400229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,6144,0.21225172678629556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,1536,0.06962239742279053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,1536,0.06754666964213053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,7168,0.2590282599131266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,1024,0.04883306821187337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,1024,0.053846398989359535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,1024,0.049983998139699296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,768,0.03932693401972453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,768,0.04605546792348226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,5120,0.18489066759745282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,512,0.031678932905197146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,512,0.03919359842936198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,4096,0.14915626843770344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,256,0.021760000785191854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,256,0.03217173417409261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,3584,0.12951786518096925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,128,0.01657919983069102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,3072,0.11006506284077962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,2560,0.09948693116505941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,64,0.014615466197331747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,64,0.02977493405342102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,6144,32,0.015478400389353433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,32,0.029290666182835896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,2048,0.08123839696248372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,1536,0.06142826477686564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,6144,128,0.0293503999710083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,768,0.040245334307352706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,65536,1.7656885782877605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,512,0.03223573366800944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,256,0.02706666588783264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,65536,2.5195327758789063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,6144,128,0.022848000129063926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,51200,1.957419713338216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,16384,0.5699455897013347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,16384,0.4452992121378581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,51200,1.3857332865397134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,12288,0.42483841578165693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,10240,0.35032854080200193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,12288,0.36376959482828775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,10240,0.3016405423482259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,8192,0.24083414077758789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,8192,0.3015189488728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,7168,0.24756693840026855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,7168,0.22533226013183594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,6144,0.2299882729848226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,6144,0.18763413429260253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,12288,0.3530069351196289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,5120,0.18393920262654623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,5120,0.16265920003255208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,65536,2.0211040496826174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,4096,0.176199467976888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,6144,0.188973871866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,16384,0.4591786702473958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,4096,0.12516799767812092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,3584,0.12709973653157552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,10240,0.31345065434773767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,3584,0.11242667039235432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,3072,0.11045440038045247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,3072,0.09983999729156494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,7168,0.20916479428609214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,2560,0.094651730855306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,2560,0.08706239859263101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,51200,1.6185279846191407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,8192,0.23081706364949545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,2048,0.09933013121287028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,2048,0.07372907002766928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,1536,0.05955413182576498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,1536,0.05518399874369303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,1024,0.0389792005221049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,1024,0.04284266630808513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,3072,0.0895903984705607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,768,0.030616533756256104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,768,0.03731093406677246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,1536,0.051115731398264565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,512,0.022473599513371786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,4096,0.12645973364512125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,512,0.031990400950113934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,256,0.01600320041179657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,256,0.027734400828679402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,5120,0.15165333747863768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,128,0.013446399569511413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,2560,0.08150400320688883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,128,0.025692800680796306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,64,0.013201066851615905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,64,0.027116799354553224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,5120,32,0.014447999993960061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,5120,32,0.027398399511973065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,3584,0.10667306582132977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,2048,0.06601066589355468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,1024,0.04309440056482951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,65536,1.4916192372639974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,65536,1.9025877634684245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,65536,1.3818197886149088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,51200,1.4874219258626302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,51200,1.1737237294514975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,16384,0.46404374440511065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,51200,1.1469557444254557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,16384,0.34612906773885094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,12288,0.3712373415629069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,16384,0.42885332107543944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,12288,0.3299605369567871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,12288,0.26956586837768554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,10240,0.2950677235921224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,10240,0.25126293500264485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,10240,0.23197013537089028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,8192,0.22228479385375977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,8192,0.17570667266845702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,8192,0.21961493492126466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,7168,0.1888202667236328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,6144,0.17816960016886393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,256,0.02216106653213501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,6144,0.15918827056884766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,5120,0.13546133041381836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,5120,0.1297386646270752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,128,0.01806079943974813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,5120,0.11245013078053792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,4096,0.1111850659052531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,3584,0.09828693072001139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,4096,0.10227733453114826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,3584,0.08264319896697998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,3584,0.09492479960123698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,3072,0.08429546356201172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,3072,0.08408426443735759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,3072,0.07190612951914468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,2560,0.07196479638417562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,512,0.026601600646972656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,2560,0.07272319793701172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,7168,0.1944287935892741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,2048,0.05880106687545776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,2048,0.06187413136164347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,2048,0.05596373478571574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,1536,0.04602453311284383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,1024,0.03295253316561381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,1536,0.045637333393096925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,1536,0.05019199848175049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,5120,768,0.03281173308690389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,1024,0.03934933344523112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,768,0.025973333915074663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,1024,0.03587413231531779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,768,0.03395306666692098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,512,0.0199562668800354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,6144,0.1490282694498698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,768,0.031057065725326537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,512,0.029618134101231892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,512,0.025624533494313557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,128,0.010708266496658325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,256,0.025835732618967693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,128,0.0233407994111379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,64,0.00909866690635681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,128,0.019450666507085164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,32,0.009336533149083455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,64,0.022674133380254112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,4096,32,0.022223999102910362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,4096,0.09229013125101725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,2560,0.06350186665852865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,4096,256,0.013989333311716715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,7168,0.1533258597056071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,65536,1.4405707041422526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,65536,1.6157834370930988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,16384,0.40412371953328446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,16384,0.4001354535420735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,51200,1.346133295694987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,12288,0.31734399795532225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,51200,1.109655507405599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,12288,0.31342827479044594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,12288,0.28610452016194665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,10240,0.2507008075714111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,10240,0.2498762607574463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,8192,0.20320746103922524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,8192,0.2048522631327311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,4096,256,0.02136853337287903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,7168,0.17940160433451335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,7168,0.16945279439290364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,6144,0.15137813886006674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,6144,0.14768959681193033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,5120,0.12652053038279215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,5120,0.12366507053375245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,65536,1.447915776570638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,4096,0.1137994686762492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,4096,0.1138698657353719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,16384,0.34859838485717776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,3584,0.0940831979115804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,51200,1.1076042175292968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,3584,0.09696853160858154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,3072,0.08040106296539307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,10240,0.2159829298655192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,3072,0.08498133023579915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,2560,0.0678762674331665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,8192,0.1742037296295166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,2560,0.0738602638244629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,2048,0.055377066135406494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,2048,0.06296746730804444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,7168,0.15587414105733235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,1536,0.043610668182373045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,1536,0.05012586514155069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,6144,0.1325269301732381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,1024,0.03151893417040507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,1024,0.039539198080698654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,5120,0.11193386713663737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,768,0.02558506727218628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,768,0.035259731610616046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,4096,0.0919157346089681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,512,0.0201855997244517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,512,0.03038933277130127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,3584,0.08087893327077231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,256,0.014505599935849508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,3072,0.06943573156992594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,256,0.02656960090001424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,128,0.011758933464686077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,128,0.02378986676534017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,2560,0.061356798807779944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,64,0.0102783997853597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,64,0.023196800549825033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3584,32,0.010844799876213073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,2048,0.05136426687240601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3584,32,0.023517866929372154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,1536,0.039995733896891275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,1024,0.03228373328844707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,768,0.025858134031295776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,65536,1.2621461232503255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,65536,1.4566752115885415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,51200,1.0408234914143881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,16384,0.3700021425882975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,16384,0.3498943964640299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,512,0.020798933506011964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,12288,0.2606869379679362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,12288,0.2508032004038493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,16384,0.34223041534423826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,10240,0.20352959632873535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,10240,0.23022185961405434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,256,0.016923733552296958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,8192,0.17166399955749512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,8192,0.1715808073679606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3584,128,0.01437226633230845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,7168,0.1425557295481364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,8192,0.16979093551635743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,7168,0.15201279322306316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,6144,0.12816639741261798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,7168,0.14664106369018554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,51200,0.9906837463378906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,5120,0.10383573373158772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,6144,0.13100372950236003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,5120,0.10484373569488525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,5120,0.1117674668629964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,4096,0.08576107025146484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,3584,0.07568106651306153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,4096,0.09262293179829915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,3584,0.08251307010650635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,3072,0.06574933528900147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,3072,0.07323946952819824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,3072,0.06432426770528157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,65536,1.283447519938151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,2560,0.06569813489913941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,2560,0.07041066487630208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,3584,0.07525226275126139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,2048,0.04543573458989461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,2048,0.05379306475321451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,2048,0.047533865769704184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,1536,0.03551359971364339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,12288,0.2392031987508138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,51200,1.0567551930745442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,1536,0.04320746660232544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,1024,0.030105600754419964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,1024,0.03375039895375569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,768,0.020032000541687012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,1024,0.034460798899332686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,768,0.02816213369369507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,512,0.01699733336766561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,512,0.027507199843724565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,512,0.020253866910934448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,256,0.011852799852689107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,256,0.024551467100779215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,256,0.01679253379503886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,6144,0.12452586491902669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,128,0.010122666756312054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,128,0.022325332959493002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,64,0.009372799595197042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,64,0.022833067178726196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,3072,32,0.00974506636460622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,4096,0.08879360357920328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,32,0.02216213345527649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,10240,0.21274026234944662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,3072,768,0.030168533325195312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,65536,1.237118911743164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,65536,1.1956287384033204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,2560,0.05755199988683065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,16384,0.28772481282552087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,51200,0.9263562520345052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,16384,0.30809386571248376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,12288,0.21420373916625976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,12288,0.23545173009236656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,1536,0.03676799933115642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,10240,0.17927467028299968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,10240,0.2008394718170166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,8192,0.1441823959350586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,8192,0.16351253191630047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,51200,0.8595114390055338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,3072,128,0.013767466942469279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,7168,0.12422186533610027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,7168,0.14508479436238605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,6144,0.10844693183898926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,6144,0.12536959648132323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,65536,1.223748270670573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,5120,0.10455146630605061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,16384,0.2975125312805176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,5120,0.11431893507639568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,4096,0.08027733167012532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,4096,0.09426986376444499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,51200,0.9751221338907877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,12288,0.23047146797180176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,3584,0.06712853113810222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,3584,0.07921493053436279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,3072,0.05909440120061239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,10240,0.1932714621225993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,3072,0.05963733196258545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,3072,0.06980799833933513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,2560,0.049347201983133956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,2560,0.059952000776926674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,2048,0.04041386842727661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,2048,0.050010665257771814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,8192,0.1500373363494873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,1536,0.03159573276837667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,7168,0.13359467188517254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,1536,0.04012053410212199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,1024,0.022359466552734374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,1024,0.032596266269683837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,6144,0.11561813354492187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,768,0.01849173307418823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,768,0.029208532969156903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,5120,0.0973578691482544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,512,0.014257066448529563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,512,0.025565866629282636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,4096,0.07924160162607828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,256,0.011087999741236369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,256,0.022098133961359658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,3584,0.06902933120727539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,128,0.008412800232569377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,128,0.020602667331695558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,2560,0.05237760146458944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,64,0.007231999933719635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,64,0.020440532763799032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2560,32,0.007696000238259633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,2048,0.04282453457514445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2560,32,0.020503467321395873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,1536,0.033532798290252686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,1024,0.026638933022816974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,768,0.021527467171351115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,65536,0.8558773040771485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,51200,0.6893898646036785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,65536,1.0472533543904623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,16384,0.24375573794047037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,16384,0.27306667963663733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,51200,0.8151573181152344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,12288,0.1617311954498291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,12288,0.20633920033772787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,512,0.017684266964594523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,10240,0.13812586466471355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,256,0.014866133530934652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,10240,0.1743765354156494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,8192,0.11121493180592854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2560,128,0.012681600451469422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,8192,0.14267093340555828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,7168,0.09792106946309408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,7168,0.1264629364013672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,7168,0.11370773315429687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,6144,0.08425280253092447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,6144,0.11009600162506103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,65536,0.9913194656372071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,5120,0.07166720231374105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,5120,0.09982826709747314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,51200,0.7634442647298176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,4096,0.05646400054295858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,16384,0.24532693227132163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,3584,0.0498634656270345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,3584,0.06763839721679688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,12288,0.1800341288248698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,3072,0.04258773326873779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,3072,0.05821119944254557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,10240,0.16090346972147623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,2560,0.03576320012410482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,2560,0.05014400084813436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,8192,0.12348159948984783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,2048,0.029140265782674153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,4096,0.07891413370768229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,2048,0.04097386598587036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,1536,0.022721066077550253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,1536,0.033779199918111166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,6144,0.09645866552988688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,1024,0.01680319905281067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,1024,0.02789546648661296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,5120,0.07994986375172933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,768,0.013393066326777139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,768,0.02384213407834371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,4096,0.06722453435262045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,512,0.009914666414260864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,512,0.021623466412226358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,3584,0.05720853408177694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,256,0.007245866457621257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,256,0.019458132982254028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,3072,0.04911253452301025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,128,0.006409599880377452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,128,0.018913066387176512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,2560,0.044310398896535236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,64,0.0060138667623202005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,64,0.018487467368443807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,2048,32,0.006437333424886067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,2048,32,0.01862186590830485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,2048,0.03582613468170166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,1536,0.027526400486628216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,65536,0.6512032190958659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,1024,0.022835199038187662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,65536,0.9385461171468099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,51200,0.5319274584452311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,16384,0.17292906443277994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,768,0.01789120038350423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,51200,0.7397493362426758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,16384,0.2579989274342855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,51200,0.6130805333455404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,12288,0.12477440039316814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,12288,0.1939583937327067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,10240,0.10479573408762614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,256,0.012040533622105916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,10240,0.16153599421183268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,8192,0.08363626797993978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,8192,0.13303360144297283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,7168,0.07325973510742187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,7168,0.11795413494110107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,128,0.010145066181818645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,6144,0.06294293403625488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,6144,0.10085972944895427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,6144,0.07995093663533528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,5120,0.053124264876047766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,5120,0.08455680211385092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,2048,512,0.0144896000623703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,4096,0.04364800055821737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,4096,0.06907413005828858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,4096,0.05354559818903605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,3584,0.03878506819407145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,3584,0.06132800181706747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,65536,0.7746442794799805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,3072,0.03790080149968465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,3072,0.0577461322148641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,16384,0.19785386721293133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,2560,0.02892373402913411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,2560,0.04486293395360311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,12288,0.1532042662302653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,2048,0.02424959937731425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,10240,0.13466560045878093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,2048,0.03872533241907756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,1536,0.01971093416213989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,2048,0.030458666880925495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,1536,0.032580266396204635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,8192,0.10451412995656331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,1024,0.014502400159835815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,1024,0.02581546703974406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,7168,0.09341333707173666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,768,0.012090667088826498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,768,0.024552534023920693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,5120,0.0657034675280253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,512,0.00951039989789327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,512,0.021644800901412964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,3584,0.046987732251485184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,256,0.0073642666141192125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,256,0.019062399864196777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,3072,0.04182933171590169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,128,0.00610453337430954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,128,0.018186666568120322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,128,0.009335466225941976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,64,0.005453866720199585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,64,0.017848533391952515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,2560,0.035571201642354326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1536,32,0.00547733356555303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1536,32,0.01761386593182882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,1536,0.02394346594810486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,65536,0.42612158457438154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,1024,0.018517333269119262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,51200,0.35164480209350585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,65536,0.8231957117716471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,16384,0.11689279874165852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,768,0.016116266449292503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,51200,0.6530815760294597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,16384,0.21471892992655434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,12288,0.08207039833068848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,10240,0.06890772978464763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,12288,0.16730772654215495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,10240,0.14121707280476886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,8192,0.055386666456858316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,8192,0.11584853331247966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,512,0.01288106640179952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,7168,0.049021867911020915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,7168,0.10303680102030437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1536,256,0.010986666878064473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,6144,0.043136000633239746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,6144,0.08974613348642985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,12288,0.13732693990071615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,5120,0.0356661319732666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,5120,0.07523093223571778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,65536,0.6971658706665039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,4096,0.02972266674041748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,4096,0.06057279904683431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,10240,0.11721279621124267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,6144,0.06920320192972819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,3584,0.026229333877563477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,3584,0.052585601806640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,3072,0.022711465756098427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,3072,0.04542080163955688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,51200,0.5496576309204102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,2560,0.01943999926249186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,2560,0.03939733505249023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,7168,0.08064959843953451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,2048,0.016212266683578492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,8192,0.09161919752756754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,16384,0.18008426030476887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,2048,0.03270080089569092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,1536,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,1536,0.027180800835291546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,1024,0.009922132889429728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,1024,0.023268266518910726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,3072,0.03684693177541097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,768,0.008668800195058186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,768,0.021908267339070638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,1536,0.01959786613782247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,512,0.007134933272997539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,512,0.019781333208084107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,5120,0.05797546704610189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,256,0.005704533557097117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,4096,0.047434667746226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,256,0.017893334229787193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,128,0.004917333523432413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,128,0.017127466201782227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,2560,0.030767999092737836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,64,0.004499199986457825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,3584,0.04164479970932007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,64,0.01685546636581421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,1024,32,0.004780800143877665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,1024,0.015597866972287497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,1024,32,0.016829866170883178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,65536,0.3435018539428711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,2048,0.02481493353843689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,65536,0.4760213216145833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,51200,0.27171198527018225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,65536,0.7776106516520183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,16384,0.08814187049865722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,16384,0.13656853040059408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,51200,0.6119722366333008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,16384,0.20296640396118165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,12288,0.06693013509114584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,12288,0.09497173627217612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,10240,0.056041598320007324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,12288,0.1575210730234782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,10240,0.08481386502583822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,8192,0.04404693444569906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,10240,0.133622407913208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,8192,0.11025813420613606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,7168,0.039027198155721025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,7168,0.09744426409403482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,51200,0.3764597256978353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,6144,0.034281599521636966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,128,0.007621333499749501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,256,0.009045333663622538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,6144,0.0848906675974528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,6144,0.05050133466720581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,5120,0.029077333211898804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,5120,0.0433514674504598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,4096,0.023396267493565878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,4096,0.03658239841461182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,4096,0.05549546480178833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,3584,0.020835200945536293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,3584,0.03237760066986084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,3584,0.048257064819335935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,8192,0.06499199867248535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,3072,0.018438400824864705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,2560,0.016201600432395935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,3072,0.04147306680679321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,2560,0.03588586648305257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,2048,0.013426132996877036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,2048,0.030333866675694782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,1536,0.011098666985829671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,512,0.010753066341082255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,7168,0.05839466651280721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,1536,0.027109332879384357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,1024,0.008583466211954754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,1536,0.017403733730316163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,1024,0.013964800039927163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,768,0.007421866556008657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,1024,0.022360533475875854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,512,0.006363733112812043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,768,0.020807466904322305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,512,0.010856533050537109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,512,0.01958293318748474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,256,0.005093333125114441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,256,0.017926400899887084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,128,0.004507733384768168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,128,0.01689066688219706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,128,0.008208000163237254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,64,0.004385066529115042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,5120,0.07116373380025229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,64,0.01683733264605204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,768,32,0.004533333579699198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,768,32,0.016535466909408568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,65536,0.23130559921264648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,1024,768,0.013267200191815695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,65536,0.7368085225423177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,51200,0.5786816279093425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,3072,0.02826346755027771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,2560,0.024529065688451132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,16384,0.06077653169631958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,2048,0.02071040074030558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,51200,0.1831391970316569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,12288,0.05288533369700114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,16384,0.13280746936798096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,16384,0.19550399780273436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,10240,0.049719464778900144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,12288,0.09929172992706299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,12288,0.15261972745259603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,8192,0.031864533821741745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,768,0.012461866935094197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,10240,0.13177599906921386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,768,256,0.009034666419029235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,8192,0.06882239977518717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,7168,0.03135573267936707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,8192,0.10576426982879639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,6144,0.028468267122904463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,7168,0.09390506744384766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,7168,0.060507734616597496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,5120,0.02547946572303772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,6144,0.05181119839350382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,4096,0.019126399358113607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,5120,0.06675946712493896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,4096,0.051362133026123045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,3584,0.017191465695699057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,3584,0.04421333471934001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,3584,0.03261546691258748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,3072,0.01578133304913839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,3072,0.03851840098698934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,3072,0.028138667345046997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,2560,0.013125333189964294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,2560,0.03328319986661275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,2048,0.011453866958618164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,2048,0.0282858669757843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,65536,0.4961130777994792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,1536,0.009160533547401428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,1536,0.02542933424313863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,6144,0.0813045342763265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,1536,0.01618666648864746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,1024,0.006681600213050842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,1024,0.021615999937057494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,768,0.006109866499900818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,768,0.01978879968325297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,51200,0.3877877235412598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,512,0.005109333495299021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,10240,0.08490666548411051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,512,0.018125865856806436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,512,0.008918399612108868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,5120,0.04470293521881104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,256,0.004287999868392944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,256,0.017211733261744182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,128,0.0038272000849246977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,128,0.016127999623616537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,2560,0.024051199356714882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,64,0.0035989334185918174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,64,0.016219733158747356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,512,32,0.0036789332826932275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,2048,0.020489599307378134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,512,32,0.01602026621500651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,65536,0.14384746551513672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,4096,0.03776106834411621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,1024,0.01262399951616923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,51200,0.11522453625996906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,768,0.010959999759991963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,16384,0.04375466505686442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,16384,0.18893866539001464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,51200,0.5463018417358398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,16384,0.12444159984588624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,12288,0.035946667194366455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,12288,0.1461141268412272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,10240,0.03234986662864685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,10240,0.12368960380554199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,10240,0.08035306930541992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,8192,0.02728853424390157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,8192,0.0996298631032308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,7168,0.022380799055099487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,65536,0.6912448247273763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,7168,0.08737493356068929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,6144,0.018129066626230875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,256,0.007623466849327088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,6144,0.07488533655802408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,5120,0.01560640037059784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,5120,0.062108798821767175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,512,128,0.006514133512973785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,4096,0.011901866396268208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,4096,0.04678080081939697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,65536,0.4705429395039876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,3584,0.010401067137718201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,3584,0.040411734580993654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,51200,0.37110932668050134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,3072,0.009462400277455648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,3072,0.03490986824035645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,12288,0.0941098690032959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,2560,0.009484799702962239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,2560,0.031221334139506025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,8192,0.06540053288141887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,2048,0.007999999821186066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,2048,0.027135999997456868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,7168,0.0577951987584432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,1536,0.006842666864395141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,1536,0.023436800638834635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,6144,0.050513064861297606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,1024,0.005351466437180838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,1024,0.02153173287709554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,5120,0.04265706539154053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,768,0.0047882666190465295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,4096,0.03369919856389363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,768,0.019218132893244425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,512,0.004167466859022776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,512,0.01781760056813558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,3584,0.029819732904434203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,256,0.0037248000502586366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,256,0.01646506687005361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,256,0.00662720004717509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,128,0.003432533393303553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,128,0.015698132912317912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,3072,0.026231465737024943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,64,0.0031658666829268134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,64,0.014985600113868713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,256,32,0.0033333333830038703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,256,32,0.015615999698638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,2560,0.02211839954058329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,65536,0.1307978630065918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,2048,0.018914133310317993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,65536,0.44026667277018233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,51200,0.10382186571756999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,1536,0.015160533785820007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,65536,0.682366943359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,16384,0.03913493156433105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,1024,0.011490133404731751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,768,0.009883733590443929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,12288,0.03123840093612671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,16384,0.18409493764241536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,12288,0.14068053563435873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,10240,0.027432533105214436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,51200,0.5372778574625652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,512,0.008039466540018718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,8192,0.022990934054056802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,10240,0.07472426891326904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,7168,0.01469546655813853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,8192,0.09725226561228434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,7168,0.08497280279795329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,6144,0.012750933567682901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,6144,0.07268906434377034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,256,128,0.005714133381843567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,10240,0.11884373029073078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,5120,0.011196800072987874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,5120,0.05817279815673828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,51200,0.3444927851359049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,16384,0.11516053676605224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,4096,0.009594666957855224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,12288,0.0893280029296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,4096,0.043738667170206705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,3584,0.008610133330027263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,3584,0.02802986701329549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,3584,0.03752640088399251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,3072,0.007975466549396515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,8192,0.06093546549479166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,3072,0.03426026503245036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,2560,0.007187200089295705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,7168,0.053710933526357016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,2048,0.006377600133419037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,2560,0.031129600604375203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,2048,0.026717867453893023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,1536,0.005712000032265982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,6144,0.04639039834340413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,1536,0.023064533869425453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,1024,0.004760533571243286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,1024,0.02018453280131022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,5120,0.039494399229685465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,768,0.004270933568477631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,768,0.009387733538945515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,768,0.01845226685206095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,512,0.003828266759713491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,512,0.01768640081087748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,256,0.0034080001215140024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,256,0.016339199741681416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,4096,0.03231253425280253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,128,0.0031498665610949195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,128,0.015396266182263692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,3072,0.024215465784072875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,64,0.002951466788848241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,2560,0.020917334159215293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,128,32,0.0030517332255840302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,64,0.014829867084821067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,128,32,0.014973866939544677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,2048,0.01769599914550781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,65536,0.12361386617024739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,1536,0.014094932874043783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,51200,0.0968618631362915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,16384,0.035580801963806155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,1024,0.011353600025177001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,12288,0.02813226580619812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,10240,0.024685867627461753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,16384,0.18094293276468915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,12288,0.13900373776753744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,8192,0.019962666432062785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,10240,0.11783466339111329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,51200,0.5312021255493165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,8192,0.09589013258616129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,6144,0.011569066842397054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,5120,0.010265599687894185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,7168,0.08358613650004068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,6144,0.07194879849751791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,4096,0.008616532882054646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,5120,0.05652693510055542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,3584,0.007937066753705342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,3072,0.007209600011507671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,4096,0.04272960027058919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,3584,0.03657066822052002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,2560,0.0065098668138186145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,3072,0.032041599353154496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,2048,0.005704533557097117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,2560,0.02874559958775838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,1536,0.005178666611512502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,2048,0.025561600923538208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,1024,0.004373333354791006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,1536,0.02256106734275818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,768,0.003920000046491623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,1024,0.01999893387158712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,768,0.018730666240056357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,256,0.003123199939727783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,512,0.016785067319869996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,128,0.00296426663796107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,256,0.01620586713155111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,64,0.002738133321205775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,64,0.014826666315396628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,512,0.007768533130486806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,32,0.0027903998891512555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,32,0.015603199601173401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,256,0.006543999910354615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,65536,0.12625066439310711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,51200,0.10037759939829509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,16384,0.03427520195643107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,16384,0.17946772575378417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,65536,0.6775157292683919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,7168,0.01360106666882833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,12288,0.027237333854039508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,10240,0.023587199052174886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,8192,0.01797440052032471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,51200,0.5293183962504069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,12288,0.1378453254699707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,10240,0.11673386891682942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,65536,0.6766304016113281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,7168,0.011146666606267293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,6144,0.010019200046857198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,5120,0.009127466877301534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,8192,0.09496320088704427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,7168,0.08256959915161133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,6144,0.07028373082478842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,4096,0.00783253312110901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,3584,0.007483733197053273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,4096,0.04233386516571045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,3072,0.007067733506361644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,3584,0.03620586792627971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,2560,0.006332799792289734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,3072,0.03317866722742717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,2048,0.005568000177542368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,64,512,0.003538133452335993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,2560,0.02959573268890381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,1536,0.004858666658401489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,1024,0.00411520004272461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,2048,0.025438932577768962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,1536,0.02299519975980123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,768,0.0037952000896135964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,1024,0.020932267109553017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,512,0.003469866762558619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,768,0.018702934185663857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,256,0.003147733211517334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,512,0.016829866170883178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2560,128,128,0.0054730668663978575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,64,128,0.0151936004559199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,256,0.01604159971078237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,64,0.00276799996693929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,128,0.015571199854214988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,32,0.0028405333558718365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,64,0.015225600202878317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,32,0.014733866850535075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2560,32,5120,0.056435199578603115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2560,32,128,0.0029343999922275543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,16384,3.834325408935547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,12288,2.607256571451823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,16384,5.583004760742187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,12288,4.965529378255209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,16384,6.707713317871094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,12288,3.766815948486328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,10240,2.3583615620930987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,10240,3.089501953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,10240,4.1901397705078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,8192,1.8568714141845704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,8192,3.3667050679524744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,8192,2.351554107666016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,7168,1.5602474212646484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,51200,11.211332194010417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,7168,2.1668576558430988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,7168,2.9468373616536456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,6144,1.3796981811523437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,6144,3.1559071858723957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,6144,1.7497834523518878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,5120,1.436138661702474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,5120,1.7550635019938152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,5120,2.7081578572591143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,4096,2.1550229390462237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,4096,1.3435551961263021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,3584,0.9957802454630533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,3584,1.0796277364095053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,3584,1.696681594848633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,4096,1.0911978403727214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,3072,0.8879061381022135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,3072,1.4430346171061197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,2560,0.6536511739095052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,3072,0.9436426798502604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,2560,0.7210847854614257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,2048,0.5317407925923665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,2560,1.0746507008870443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,2048,0.9971882502237955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,1536,0.38154452641805015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,1536,0.6867658615112304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,1024,0.34481067657470704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,2048,0.7376799901326497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,1024,0.506498146057129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,1536,0.5460383733113606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,768,0.3710474650065104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,512,0.18189226786295573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,512,0.2849952061971029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,768,0.3833770751953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,256,0.16056747436523439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,256,0.1555093288421631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,128,0.13765865961710613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,512,0.33020906448364257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,128,0.14137600262959799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,64,0.12699519793192546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,128,0.22500054041544595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,32,0.1241919994354248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,64,0.1378709316253662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,32,0.13424320220947267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,65536,768,0.2727146784464518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,256,0.2417450745900472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,51200,23.530729166666667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,65536,51200,22.681982421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,16384,5.820500183105469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,65536,1024,0.37255786259969076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,51200,9.13487040201823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,16384,3.113395182291667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,65536,11.86885986328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,16384,4.050687917073568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,12288,2.2567593892415365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,12288,4.797559611002604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,12288,2.86727778116862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,10240,1.9753162384033203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,51200,18.444757080078126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,10240,3.2770154317220053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,10240,2.289280954996745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,8192,2.653326924641927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,8192,1.5036373138427734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,51200,16.65699971516927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,7168,1.3154037475585938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,8192,1.8357877095540363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,7168,2.375836690266927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,7168,1.6737300872802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,6144,1.0554389317830404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,6144,1.3922442118326823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,5120,0.9391935984293619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,65536,21.78168741861979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,5120,1.633181889851888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,5120,1.349078369140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,4096,0.7744490941365559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,4096,1.4356075286865235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,3584,0.6472447713216145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,4096,1.0033248265584311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,6144,2.1750516255696613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,3584,1.179375966389974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,65536,23.10165812174479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,3584,0.9632458368937173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,3072,0.6132160186767578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,3072,0.7338399887084961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,3072,1.055058161417643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,2560,0.9069354375203451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,2560,0.4690538724263509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,2048,0.4307104110717773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,2560,0.5673429489135742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,2048,0.5751904169718425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,1024,0.38939199447631834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,1536,0.5394304275512696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,1024,0.24703787167867025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,1536,0.4577141443888347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,512,0.20411307017008462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,1024,0.3119797388712565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,768,0.21315946578979492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,768,0.3125130653381348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,512,0.15147520701090494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,256,0.12775253454844157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,256,0.12713812987009684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,128,0.11051092942555744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,256,0.2149066607157389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,128,0.11459626356760662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,64,0.10141867001851398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,128,0.19527146021525066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,32,0.09923946857452393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,64,0.11395839850107829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,32,0.11277120113372803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,51200,1536,0.3584800084431966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,2048,0.6463210423787434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,51200,768,0.2803253491719564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,51200,512,0.2369919935862223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,16384,0.9514538447062174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,16384,1.6455477396647136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,16384,1.2044053395589194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,51200,3.165583038330078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,12288,1.3174517313639322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,65536,4.1569770812988285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,12288,0.7414655685424805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,12288,0.8704906463623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,10240,1.0321717580159506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,10240,0.6492991765340169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,51200,5.286326599121094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,8192,0.5037952105204264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,8192,0.5958656311035156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,65536,6.052790323893229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,8192,0.8613728205362955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,7168,0.44086294174194335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,7168,0.7384757359822591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,7168,0.5438079833984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,6144,0.40627307891845704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,51200,4.40851084391276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,6144,0.6549983978271484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,6144,0.44619201024373367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,65536,6.733607482910156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,5120,0.3258048057556152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,5120,0.5197194735209147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,5120,0.3810655911763509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,4096,0.28417812983194984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,4096,0.313814385732015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,3584,0.23083839416503907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,3584,0.2887946764628092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,3072,0.31206932067871096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,3584,0.3946282704671224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,3072,0.21230613390604652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,2560,0.1832863966623942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,2560,0.2477440039316813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,2048,0.14932266871134442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,2560,0.216759459177653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,1536,0.16516373952229818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,2048,0.18521706263224286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,1536,0.11699946721394856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,1536,0.14978666305541993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,1024,0.11769813696543377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,1024,0.09514026641845703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,768,0.09269013404846191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,1024,0.11146453221638997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,512,0.06684906482696533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,768,0.08088533083597818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,768,0.10490026473999023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,512,0.059854932626088465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,256,0.04377919832865397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,512,0.08982613086700439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,256,0.05119040012359619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,256,0.06911253134409587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,128,0.0468938668568929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,128,0.06357760032018026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,4096,0.3923274676005045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,64,0.033403734366099044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,32,0.034358398119608564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,32,0.048000001907348634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,2048,0.2311946709950765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,16384,128,0.037257599830627444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,16384,64,0.04828799962997436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,65536,3.1960606892903645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,10240,0.7225311915079753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,51200,2.45391362508138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,65536,4.39487559000651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,16384,1.2774954477945963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,16384,0.8102133433024088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,16384,0.8690442403157552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,65536,5.053687540690104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,51200,3.9848459879557288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,12288,0.6012639999389648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,12288,0.6587306976318359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,51200,2.925548807779948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,12288,1.0127285639444987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,10240,0.5119850794474284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,10240,0.5478431701660156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,10240,0.8501344045003256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,8192,0.43857920964558916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,8192,0.6687797546386719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,7168,0.3649322509765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,8192,0.475164794921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,6144,0.49234453837076825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,7168,0.5733386357625325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,6144,0.32192961374918616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,6144,0.3863914807637533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,5120,0.39545065561930337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,5120,0.27558078765869143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,5120,0.28851518630981443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,4096,0.21488533020019532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,4096,0.34449920654296873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,3584,0.19202879269917805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,4096,0.26415467262268066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,3584,0.28643093109130857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,3584,0.2114528020222982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,3072,0.2344202677408854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,3072,0.18208853403727215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,2560,0.14942080179850262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,2560,0.2218997319539388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,2560,0.15849067370096842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,2048,0.16428267161051432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,2048,0.12148479620615642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,2048,0.13848212560017903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,1536,0.1419370651245117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,1536,0.09816853205362955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,1536,0.11286826928456624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,1024,0.09037439823150635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,768,0.07147839864095053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,768,0.06559253136316935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,512,0.05375786622365316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,768,0.07358400026957193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,512,0.055452799797058104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,256,0.036909866333007815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,512,0.060850131511688235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,7168,0.39692586263020835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,256,0.05125333468119303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,128,0.030333866675694782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,256,0.044354132811228436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,128,0.04754240115483602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,64,0.02712533275286357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,12288,32,0.02685653368631999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,64,0.039596800009409586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,32,0.04096213181813558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,3072,0.1815338611602783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,1024,0.07541653315226236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,16384,3072,0.23060266176859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,12288,128,0.03954773346583049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,16384,1.0475744247436523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,12288,1024,0.08164479732513427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,65536,2.7054400126139324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,51200,2.1114133199055987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,16384,0.682362683614095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,65536,3.3314900716145837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,51200,2.4480255126953123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,16384,0.7660032272338867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,12288,0.8115626653035483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,12288,0.5142709414164226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,12288,0.5665109634399415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,51200,3.527806854248047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,10240,0.5022186597188314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,8192,0.35351680119832357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,10240,0.7142741521199544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,65536,4.27569580078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,8192,0.5426730473836263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,8192,0.4122645378112793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,7168,0.4712245305379231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,7168,0.3035327911376953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,7168,0.3449749310811361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,6144,0.2683007876078288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,6144,0.40818347930908205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,6144,0.3068810780843099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,5120,0.22136425971984863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,5120,0.3489568074544271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,5120,0.24664640426635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,4096,0.256057596206665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,4096,0.19167253176371257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,4096,0.21728533109029136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,3584,0.22243839899698892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,3584,0.17834986050923665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,3584,0.17757333119710286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,3072,0.1494965394337972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,3072,0.16107734044392902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,2560,0.16771413485209147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,2560,0.1231274684270223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,2560,0.13820692698160808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,2048,0.11004160245259602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,2048,0.12110506693522136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,1536,0.10219093163808186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,1536,0.08983360131581625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,1024,0.07731413046518962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,10240,0.45084053675333663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,1024,0.06753280162811279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,768,0.06529813210169474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,1024,0.0728437344233195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,768,0.06288640101750692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,512,0.04979413350423177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,512,0.04912000099817912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,256,0.03349546591440837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,256,0.04004266659418742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,128,0.026947200298309326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,128,0.036806400616963705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,3072,0.19621866544087727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,64,0.023423999547958374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,64,0.0371829350789388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,32,0.026044799884160356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,32,0.03784213463465373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,10240,768,0.05858773390452067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,10240,2048,0.13249279657999674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,1536,0.08993919690450033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,512,0.053220268090566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,256,0.04404160181681315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,10240,128,0.04113920132319133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,65536,2.1579851786295574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,16384,0.758888562520345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,16384,0.554043706258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,65536,3.44300791422526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,16384,0.8395605087280273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,12288,0.6303082784016927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,12288,0.4372085253397624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,51200,1.7398571014404296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,51200,2.7133494059244794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,10240,0.3815349260965983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,10240,0.537060292561849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,8192,0.4060501416524251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,8192,0.29777386983235676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,7168,0.26219627062479656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,51200,2.7515828450520834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,6144,0.2973973274230957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,6144,0.23773226737976075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,5120,0.26451199849446616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,5120,0.19055360158284504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,7168,0.38084799448649087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,4096,0.19786453247070312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,4096,0.14853973388671876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,4096,0.2034229278564453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,3584,0.17496320406595867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,3584,0.13326400121053059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,12288,0.5867274602254231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,10240,0.49483518600463866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,3072,0.12076480388641357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,3072,0.16545173327128093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,8192,0.39021546045939126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,2560,0.14177600542704266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,2560,0.10192639827728271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,2048,0.10305600166320801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,65536,4.099497477213541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,2048,0.08660693168640136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,7168,0.35672105153401695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,1536,0.07993173599243164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,1536,0.07151786486307779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,1024,0.057036801179250085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,1024,0.0563594659169515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,1024,0.06084693272908529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,768,0.046326398849487305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,6144,0.2950922648111979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,768,0.04874560038248698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,512,0.035675732294718425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,512,0.04055573145548503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,5120,0.244541867574056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,256,0.024269866943359374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,3584,0.17509652773539225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,256,0.03393813371658325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,256,0.031940267483393354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,128,0.019332265853881835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,128,0.033073065678278606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,3072,0.14774293899536134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,64,0.019705599546432494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,64,0.03375680049260457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,2560,0.127948800722758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,8192,32,0.020363734165827433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,8192,32,0.03377386728922526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,2048,0.10261013507843017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,1536,0.07579413255055746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,768,0.04673386812210083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,512,0.03793066740036011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,65536,2.0668715159098308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,16384,0.6967701594034831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,51200,2.4701769510904947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,51200,1.6180789947509766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,16384,0.5258367856343586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,8192,128,0.026719999313354493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,65536,3.237712097167969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,12288,0.3998304049173991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,12288,0.5369951883951823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,10240,0.4416960080464681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,10240,0.3506474812825521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,8192,0.3642271995544434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,8192,0.2657983938852946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,8192,0.35245227813720703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,7168,0.33267412185668943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,7168,0.23928319613138832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,6144,0.20747413635253906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,6144,0.3048426628112793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,7168,0.317411200205485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,5120,0.18110826810201008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,5120,0.24876160621643068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,4096,0.19988266626993817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,4096,0.15296746889750162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,16384,0.7314282735188802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,12288,0.5274079958597819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,3584,0.16678187052408855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,65536,3.412921651204427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,3584,0.13559679985046386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,51200,2.617877451578776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,3072,0.1697973410288493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,3072,0.11568426291147868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,10240,0.44383678436279295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,2560,0.1444287935892741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,2560,0.09889600276947022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,2048,0.11083093484242756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,2048,0.0844533363978068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,1536,0.0764415979385376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,1536,0.06946773529052734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,6144,0.2639231999715169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,1024,0.05409706830978393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,1024,0.05442453225453695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,5120,0.22351147333780924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,768,0.04521706501642863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,768,0.047653333346048994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,4096,0.18042772610982258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,512,0.0342741330464681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,512,0.039692799250284835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,512,0.03477226495742798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,256,0.024239999055862427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,256,0.032630399862925215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,3584,0.15593066215515136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,128,0.018338133891423546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,128,0.029346134265263873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,128,0.02490133245786031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,64,0.01562666694323222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,3072,0.1339306672414144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,7168,32,0.015923200050989787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,2560,0.11573546727498371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,32,0.02959573268890381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,2048,0.09210986296335856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,1536,0.06886826356252035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,1024,0.05464320182800293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,768,0.04285013278325399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,7168,64,0.029674667119979858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,7168,256,0.028952533006668092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,65536,1.7480971018473308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,16384,0.5749877293904622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,65536,1.9708704630533855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,16384,0.4375178654988606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,51200,1.3630655924479167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,12288,0.460427729288737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,16384,0.46880747477213547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,65536,2.6375498453776043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,12288,0.3470709482828776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,51200,1.4878196716308594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,12288,0.37889172236124674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,10240,0.31389973958333334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,10240,0.3085941314697266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,10240,0.3683904012044271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,8192,0.29837865829467775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,8192,0.23793066342671715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,7168,0.2736639976501465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,8192,0.2710634549458822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,7168,0.2169813315073649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,7168,0.2202122688293457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,6144,0.22575467427571616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,6144,0.18302933375040692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,6144,0.1846666653951009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,5120,0.19732799530029296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,5120,0.15721492767333983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,5120,0.17402346928914386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,4096,0.1282591978708903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,4096,0.1265173355738322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,3584,0.14693439801534017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,3584,0.11225919723510742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,3584,0.11828479766845704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,3072,0.11469439665476482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,3072,0.0993226687113444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,3072,0.10883946418762207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,2560,0.09560426870981852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,2560,0.09241387049357096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,2048,0.07893973191579183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,2048,0.07184533278147379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,2048,0.07957226435343424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,1536,0.06305386622746786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,1536,0.06102293332417806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,1024,0.04922026793162028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,1536,0.05871893167495727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,1024,0.050487466653188075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,1024,0.044921600818634035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,768,0.042049066225687666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,768,0.04276373386383057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,512,0.03514773448308309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,256,0.018902399142583213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,256,0.030028800169626873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,256,0.028328533967336016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,128,0.01591253379980723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,128,0.027679999669392902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,128,0.02616426746050517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,64,0.013925333817799887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,64,0.027885866165161134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,32,0.014407466848691305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,4096,0.15499839782714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,6144,32,0.027824000517527266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,768,0.03997013171513875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,512,0.027284266551335652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,512,0.03476053476333618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,65536,1.6116981506347656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,6144,2560,0.08315733273824057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,6144,51200,2.0625045776367186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,51200,1.2894752502441407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,65536,2.2939913431803385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,16384,0.5504117329915365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,65536,1.6760682423909503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,51200,1.6854624430338543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,16384,0.4098047892252604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,12288,0.4028970718383789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,12288,0.3315381368001302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,12288,0.3292682647705078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,10240,0.3733728090922038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,10240,0.2518869400024414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,10240,0.2916085243225098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,8192,0.27284053166707356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,8192,0.2372181256612142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,7168,0.18892159461975097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,51200,1.308835220336914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,7168,0.18707733154296874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,6144,0.20793600082397462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,6144,0.17345813115437825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,6144,0.1593802611033122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,5120,0.14479893048604328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,5120,0.1987178643544515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,5120,0.1453418731689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,4096,0.13320000171661378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,4096,0.1170357306798299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,4096,0.11090346972147624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,3584,0.14370452562967936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,3584,0.09981226921081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,3584,0.11798826853434245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,3072,0.10320639610290527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,3072,0.09682453473409017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,2560,0.08785920143127442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,3072,0.10681173006693523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,2560,0.08711466789245606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,2048,0.07348480224609374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,2048,0.0731381336847941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,1536,0.06059733231862387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,1536,0.053021868069966636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,1024,0.04410133361816406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,1024,0.047489066918690995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,1024,0.041568001111348465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,768,0.03580053249994914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,768,0.03674559990564982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,768,0.0404309352238973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,512,0.02714666724205017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,512,0.03482666810353597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,512,0.030330665906270343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,256,0.01951573292414347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,256,0.029433600107828778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,256,0.02529279987017314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,128,0.01463573376337687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,7168,0.2547935962677002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,128,0.022936532894770302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,128,0.02726080020268758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,64,0.012486400206883748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,32,0.013106133540471396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,64,0.026787199576695758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,5120,32,0.02617173393567403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,2560,0.07581760088602701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,5120,1536,0.06125973463058472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,16384,0.3957770665486654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,65536,1.3542986551920573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,8192,0.2027221361796061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,65536,1.715240478515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,51200,1.3057333628336587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,65536,1.3948309580485025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,16384,0.3891946792602539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,16384,0.37997334798177085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,12288,0.34028374354044594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,16384,0.36734933853149415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,51200,1.0674293518066407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,12288,0.28976319630940756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,5120,2048,0.0630944013595581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,10240,0.27707093556722007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,8192,0.20536853472391764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,8192,0.1818826675415039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,10240,0.21702399253845214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,10240,0.25242346127827964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,8192,0.18342933654785157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,7168,0.17950080235799154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,7168,0.15961066881815594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,7168,0.15597866376241049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,6144,0.14729173978169757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,6144,0.1387818654378255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,5120,0.12579306761423747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,5120,0.11854720115661621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,4096,0.10575040181477864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,5120,0.11557546456654866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,4096,0.09541760285695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,4096,0.09558506806691489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,3584,0.09347093105316162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,3072,0.07852906386057536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,3584,0.08222506841023763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,3584,0.09066239992777506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,2560,0.06572479804356893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,3072,0.07349119981129965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,2560,0.06927893161773682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,2048,0.054306133588155114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,2048,0.053922132651011145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,2048,0.05884480079015096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,1536,0.042529066403706864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,1024,0.030771199862162275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,1536,0.04705280065536499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,1024,0.034995198249816895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,1024,0.036748798688252766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,768,0.024634667237599692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,768,0.032129067182540896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,768,0.0305567999680837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,512,0.019245866934458414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,512,0.027847466866175334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,512,0.025961599747339886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,256,0.013155200084050498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,256,0.026358399788538617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,256,0.02111146648724874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,128,0.012575999895731608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,128,0.02427306572596232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,128,0.019755733013153077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,64,0.010941867033640544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,32,0.011448533336321513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,64,0.024740266799926757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,32,0.024945066372553507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,4096,3072,0.08244799772898356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,1536,0.04479680061340332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,12288,0.24934719403584799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,4096,6144,0.1592682679494222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,65536,1.2722688039143881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,65536,1.5780916849772135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,51200,1.0674911499023438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,16384,0.35222291946411133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,51200,1.1572394053141275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,16384,0.37367146809895835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,12288,0.2665162722269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,12288,0.27329387664794924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,51200,0.9854133605957032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,4096,2560,0.06250880161921182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,10240,0.24058987299601237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,10240,0.22411200205485024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,10240,0.21732692718505858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,51200,1.0965098063151042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,8192,0.1771018664042155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,8192,0.18574612935384113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,7168,0.15625173250834148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,7168,0.16536854108174642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,6144,0.13294719854990641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,5120,0.11235733032226562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,5120,0.10988693237304688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,5120,0.11024959882100423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,4096,0.09091306527455648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,4096,0.09072853724161783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,6144,0.1465173403422038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,3584,0.08119573593139648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,3584,0.08269759813944498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,65536,1.3264778137207032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,3072,0.07724693616231283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,16384,0.3336928049723307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,3072,0.08172799746195475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,12288,0.2503925323486328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,2560,0.059714134534200033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,2560,0.0645578662554423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,2560,0.060342399279276523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,2048,0.049105068047841385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,7168,0.15095465977986652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,2048,0.0549397349357605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,6144,0.13147626717885336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,1536,0.038089601198832194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,1536,0.04408106803894043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,1536,0.03949866692225139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,1024,0.027883734305699664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,1024,0.03505920171737671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,768,0.02255786657333374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,768,0.030400000015894574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,4096,0.09175466696421306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,512,0.017755732933680216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,512,0.027245867252349853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,512,0.02097066640853882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,256,0.012196266651153564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,8192,0.16922666231791178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,256,0.02481600046157837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,128,0.010637866457303365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,128,0.021975467602411904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,3584,0.07866986592610678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,64,0.008970666925112407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,64,0.02276373306910197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3584,32,0.00978773335615794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3584,32,0.022771199544270836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,3072,0.06804266770680746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,2048,0.04983359972635905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,65536,1.1683626810709635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,1024,0.03171520034472148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,768,0.02521173357963562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,65536,1.1283220926920572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,16384,0.285315195719401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,51200,0.9909727732340494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,16384,0.2943264007568359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,51200,0.902558962504069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,256,0.016458666324615477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,12288,0.24458667437235512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3584,128,0.0143477330605189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,65536,1.372991943359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,16384,0.35428266525268554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,10240,0.17891519864400227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,12288,0.25266772905985513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,12288,0.22466665903727212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,10240,0.20692906379699708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,7168,0.13058133125305177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,8192,0.1557535966237386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,8192,0.1461407979329427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,7168,0.13470293680826823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,6144,0.11262719631195069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,6144,0.11801493167877197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,5120,0.09318613211313884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,5120,0.10258560180664063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,4096,0.07568106651306153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,4096,0.08779733180999756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,3584,0.06841279665629069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,3584,0.0766101360321045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,3584,0.07997972965240478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,3072,0.05860053300857544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,3072,0.06736319859822591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,6144,0.13433067003885907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,2560,0.047890134652455646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,2560,0.05672533512115478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,51200,1.0621536254882813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,8192,0.16820906003316244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,2048,0.04474453528722127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,2048,0.04755200147628784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,1536,0.03052373329798381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,7168,0.15218666394551594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,1536,0.03857386509577433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,1536,0.03724693457285563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,1024,0.022367999951044718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,1024,0.03183786670366923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,1024,0.028700800736745198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,5120,0.10972479979197185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,768,0.018427733580271402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,768,0.027406932910283406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,512,0.014403200149536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,512,0.018761599063873292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,512,0.02401813268661499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,256,0.01000426709651947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,256,0.020988800128300986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,256,0.01585813363393148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,128,0.007933866480986278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,128,0.021053866545359293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,4096,0.09208746751149496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,128,0.013489066561063131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,64,0.007851733267307282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,3072,32,0.008345599969228108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,64,0.021016534169514975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,3072,32,0.021013333400090536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,3072,0.06734933058420817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,10240,0.21610132853190103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,65536,0.9726570765177408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,65536,0.8648778915405273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,51200,0.7784725189208984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,65536,1.0529653549194335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,2560,0.059012266000111904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,16384,0.24834559758504232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,51200,0.8124746958414713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,16384,0.22427306175231934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,12288,0.19616959889729818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,16384,0.27540906270345056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,12288,0.21105920473734535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,12288,0.1740671952565511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,10240,0.166429869333903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,10240,0.1768405278523763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,51200,0.7474847793579101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,10240,0.14572693506876627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,8192,0.154747740427653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,8192,0.11871573130289714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,7168,0.12215466499328613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,7168,0.12485973040262859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,6144,0.09574613571166993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,6144,0.11976426442464191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,5120,0.0809653361638387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,5120,0.09955093065897623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,4096,0.06702933311462403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,2048,0.0477343996365865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,4096,0.07930986881256104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,3584,0.056900266806284586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,4096,0.06410880088806152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,3584,0.0573578675587972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,3584,0.07189226945241292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,3072,768,0.02332800030708313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,3072,0.05166293382644653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,3072,0.06321173508961996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,2560,0.04466346502304077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,3072,0.050154666105906166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,2560,0.05392959912618002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,2048,0.033854933579762776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,2560,0.043987198670705156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,1536,0.027804799874623615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,2048,0.03732906579971314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,1536,0.030689066648483275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,1536,0.03580373525619507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,1024,0.019359999895095827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,8192,0.12486506303151448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,1024,0.02949333389600118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,1024,0.02464853326479594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,768,0.016132266322771708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,512,0.012342400352160136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,768,0.0268832008043925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,512,0.018051199118296304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,512,0.023727999130884806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,256,0.008577066659927367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,256,0.020188800493876138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,6144,0.09328426520029703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,256,0.015293866395950317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,128,0.00697386662165324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,128,0.013983999689420065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,64,0.006321066617965698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,128,0.019615999857584634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2560,32,0.00913279950618744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,64,0.022114133834838866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,32,0.021692800521850585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2560,2048,0.044725334644317626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,65536,0.9448874791463217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,51200,0.6087946573893229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,7168,0.10576213200887044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,51200,0.7376138687133789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,16384,0.2155797322591146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,16384,0.24982719421386718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,5120,0.0784330685933431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,12288,0.1905951976776123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,10240,0.12307626406351727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,65536,0.7960255940755208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,10240,0.1601930618286133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,8192,0.09940266609191895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,8192,0.13118933041890463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2560,768,0.021733333667119346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,7168,0.08655573527018229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,7168,0.115721599260966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,12288,0.16643840471903484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,6144,0.07667413552602133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,6144,0.10316373507181804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,65536,0.9868490854899088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,5120,0.06889920234680176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,5120,0.0917301336924235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,5120,0.08527680238087973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,4096,0.05295039812723795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,4096,0.07256213029225668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,4096,0.06776320139567057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,51200,0.7903551737467448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,16384,0.24170452753702798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,3584,0.047325865427653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,3072,0.041384534041086836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,3584,0.0663978656133016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,3072,0.0555733323097229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,12288,0.1884511947631836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,2560,0.03460266590118408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,2560,0.046486401557922365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,2048,0.027988266944885255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,2048,0.03946666717529297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,2048,0.0348906675974528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,10240,0.16224320729573566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,1536,0.022423466046651207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,1536,0.03334720134735107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,8192,0.12602026462554933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,1024,0.017035732666651406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,1024,0.027646932999293012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,7168,0.11314559777577718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,768,0.013754666845003764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,768,0.024369066953659056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,768,0.017845332622528076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,6144,0.09580159982045491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,512,0.01076479951540629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,512,0.022281599044799805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,256,0.008177066842714947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,256,0.01972800095876058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,3584,0.05619733333587647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,128,0.0068917334079742435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,3072,0.04939519961675008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,128,0.018114133675893148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,64,0.006157866617043813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,2560,0.044232531388600664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,64,0.01813653310139974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,2048,32,0.01840959986050924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,1536,0.027500800291697186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,65536,0.5657023747762044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,1024,0.021478400627772013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,51200,0.4712095896402995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,65536,0.8398911794026693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,512,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,65536,0.8113248189290365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,51200,0.6590709050496419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,16384,0.1444383939107259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,256,0.012077866991360982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,12288,0.10922239621480305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,16384,0.2192255973815918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,12288,0.17176960309346517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,10240,0.09160426457722982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,10240,0.14222079912821453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,8192,0.07402453422546387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,8192,0.11709012985229492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,7168,0.06450666586558024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,7168,0.10391146341959637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,2048,32,0.006808533271153767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,6144,0.0550378680229187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,2048,128,0.010735999544461567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,6144,0.0902122656504313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,5120,0.04658773342768351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,5120,0.07575679620107015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,12288,0.15032320022583007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,4096,0.03917440176010132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,4096,0.062286933263142906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,10240,0.1295082648595174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,8192,0.10395413239796955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,3584,0.03434453407923381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,3584,0.054542934894561766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,3072,0.029688533147176104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,3072,0.04730879863103231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,3072,0.04049280087153117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,7168,0.09165013631184896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,2560,0.025093332926432295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,2560,0.04007573525110881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,2048,0.02070080041885376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,2048,0.034777601559956864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,51200,0.6198303858439128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,6144,0.07806293169657388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,1536,0.017112533251444496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,1536,0.029090134302775066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,1024,0.01249066690603892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,5120,0.06493226687113443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,1024,0.024918399254480996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,768,0.010491733749707539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,768,0.02230293353398641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,16384,0.20265706380208334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,512,0.008548266688982646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,512,0.020190932353337607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,512,0.012612266341845193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,256,0.006446933249632518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,256,0.01853760083516439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,4096,0.05470933516820272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,128,0.005668266614278158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,128,0.017707733313242595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,3584,0.046469334761301676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,64,0.004950400193532308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,64,0.017273600896199545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1536,32,0.005165866514046987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,2560,0.03480319976806641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,2048,0.028816000620524092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,65536,0.4111488024393718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,1536,0.023678932587305704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,1024,0.01800533334414164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,51200,0.3211989402770996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,65536,0.7601439793904622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,768,0.015467733144760132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,16384,0.10459520022074383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,16384,0.2003413359324137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,16384,0.1785439968109131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,51200,0.5960842768351238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1536,32,0.01737173398335775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,12288,0.07995733420054117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,12288,0.15320852597554524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,12288,0.13413119316101074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,10240,0.06755839983622233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,10240,0.13142720063527424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,8192,0.053940268357594814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,8192,0.10746026833852132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,7168,0.0510101318359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,7168,0.09479466279347738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,256,0.010377599795659383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,6144,0.04134293397267659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,6144,0.08218453725179037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1536,128,0.009117866555849712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,5120,0.03476586739222209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,5120,0.07011306285858154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,65536,0.697105089823405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,4096,0.027138133843739826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,51200,0.5472021102905273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,4096,0.053668268521626795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,3584,0.024913066625595094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,10240,0.11395306587219238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,3584,0.046804265181223555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,3072,0.02092906634012858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,3072,0.040907732645670575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,8192,0.09102826913197835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,3072,0.03564586639404297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,2560,0.019014400243759156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,2560,0.035613866647084554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,2048,0.015209600329399109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,2048,0.03026026686032613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,1536,0.012242133418718975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,1536,0.02678080002466838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,6144,0.06949120362599691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,1024,0.009443199634552002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,1024,0.022745599349339805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,1024,0.015421866377194723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,768,0.00806826651096344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,768,0.021944532791773476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,7168,0.07956159909566243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,512,0.006837333242098491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,512,0.01969919999440511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,5120,0.058390398820241295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,256,0.0055071999629338585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,4096,0.04782933394114176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,256,0.018346667289733887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,128,0.004820266862710317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,128,0.017167999347050985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,3584,0.041569066047668454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,64,0.004299733539422353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,64,0.01690346598625183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,1024,32,0.004675200084845225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,2560,0.030920533339182536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,1024,32,0.016705065965652466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,2048,0.02416960000991821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,1536,0.01954666574796041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,65536,0.30399465560913086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,51200,0.24531839688618978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,65536,0.750378672281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,51200,0.6040309270222981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,16384,0.09031039873758952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,768,0.012970667084058127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,12288,0.08014933268229166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,16384,0.20093119939168297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,12288,0.15403520266215007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,10240,0.06681813398996989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,512,0.010656000177065531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,10240,0.07448639869689941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,8192,0.04509546756744385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,8192,0.10823360284169514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,8192,0.06114879846572876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,7168,0.0447648008664449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,7168,0.0938922643661499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,256,0.00885653297106425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,7168,0.05452693303426107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,6144,0.041764267285664874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,6144,0.08168213367462158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,6144,0.04830933411916097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,1024,128,0.007523199915885926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,5120,0.03343786795934041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,4096,0.024258132775624594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,5120,0.06717120011647543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,4096,0.052799999713897705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,4096,0.0339850664138794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,3584,0.023542400201161703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,3584,0.04594560066858928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,10240,0.13536319732666016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,3072,0.02150933345158895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,3072,0.039544534683227536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,2560,0.020124799013137816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,2560,0.03484906752904256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,2560,0.023206400871276855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,2048,0.015215999881426492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,2048,0.030455466111501055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,65536,0.4455904006958008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,1536,0.011437867085138958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,1536,0.02576746741930644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,1536,0.016565333803494772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,16384,0.11796800295511882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,1024,0.008752000331878663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,1024,0.022732800245285033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,768,0.007402666906515758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,12288,0.09013866583506266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,768,0.020230400562286376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,768,0.0121781329313914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,512,0.006017066538333893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,512,0.01900586684544881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,51200,0.3507765452067057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,256,0.004891733328501383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,256,0.017303466796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,5120,0.04148906469345093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,128,0.004312533140182495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,128,0.01639786660671234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,128,0.008089600006739299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,64,0.003868799904982249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,3072,0.026219733556111652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,64,0.016167466839154564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,768,32,0.004068266600370407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,768,32,0.016247466206550598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,65536,0.21245759328206382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,2048,0.01992853283882141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,51200,0.16901973088582356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,65536,0.668508783976237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,51200,0.37383572260538733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,1024,0.013528533776601157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,3584,0.030190932750701904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,51200,0.5283775965372721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,16384,0.05835733413696289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,512,0.010479999581972758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,12288,0.0433514674504598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,16384,0.17644267082214354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,10240,0.036772267023722334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,12288,0.09639466603597005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,12288,0.13724053700764974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,8192,0.029762132962544756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,10240,0.08258346716562906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,10240,0.11579733689626057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,7168,0.026443733771642046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,8192,0.0954314629236857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,6144,0.02227413256963094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,7168,0.08385600248972574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,6144,0.0714133342107137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,5120,0.019581866264343262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,5120,0.058616534868876136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,4096,0.016389333208402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,4096,0.04527999957402547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,768,256,0.00895146628220876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,3584,0.014748799800872802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,3584,0.039833601315816244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,65536,0.48533973693847654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,3072,0.012959999839464822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,3072,0.03520960013071696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,16384,0.12764800389607747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,2560,0.011383466919263204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,2560,0.030396799246470135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,6144,0.05064213275909424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,8192,0.06670506795247397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,2048,0.00995733340581258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,7168,0.05809599955876669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,2048,0.01964799960454305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,2048,0.027524266640345258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,1536,0.008438400427500407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,5120,0.044232531388600664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,1536,0.024184532960255942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,1024,0.0067104001839955645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,768,0.005986133217811584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,1024,0.02119999925295512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,768,0.019434666633605956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,4096,0.03577920198440552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,512,0.005201066533724466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,512,0.01840533415476481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,256,0.0043605332573254905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,256,0.01715946594874064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,256,0.007250133156776428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,128,0.004142933338880539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,128,0.015990400314331056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,3584,0.031129600604375203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,64,0.0036650667587916053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,64,0.01570026675860087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,512,32,0.003807999938726425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,512,32,0.015596800049146018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,65536,0.14019734064737957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,65536,0.6253023783365885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,3072,0.02762239972750346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,51200,0.11194240252176921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,65536,0.4354943911234538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,51200,0.49285227457682296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,16384,0.041922132174173996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,2560,0.02323306600252787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,16384,0.1686591943105062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,12288,0.030664533376693726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,1536,0.015867732961972556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,1024,0.012443733215332032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,10240,0.026293333371480303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,12288,0.0894442637761434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,12288,0.1303114652633667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,768,0.010734933614730834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,8192,0.020866133769353232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,10240,0.110099196434021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,8192,0.09007253646850585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,7168,0.01672533353169759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,512,0.008663466572761536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,7168,0.07831786473592123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,6144,0.015314132968584696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,6144,0.06678187052408854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,6144,0.04826773405075073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,512,128,0.006482133269309997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,5120,0.013006933530171714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,5120,0.053053867816925046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,4096,0.010983467102050781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,4096,0.04083733161290486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,16384,0.11521493593851725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,3584,0.010171733299891154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,3584,0.0355402668317159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,3584,0.028970666726430255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,3072,0.009170132875442504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,3072,0.031496532758076984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,51200,0.35155839920043946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,2560,0.00820906658967336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,2560,0.029024000962575274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,8192,0.061292799313863125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,10240,0.07441279888153077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,2048,0.007447466750939687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,7168,0.053600001335144046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,2048,0.026182399193445845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,1536,0.006099199752012888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,1536,0.022809600830078124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,1024,0.005276800195376078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,1024,0.02031466762224833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,1024,0.011387733618418376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,768,0.004689066608746847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,768,0.018761599063873292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,5120,0.03959999879201253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,512,0.004091733445723852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,512,0.018332799275716148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,512,0.007825066645940144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,4096,0.03327893416086833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,256,0.003537066777547201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,256,0.016773333152135216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,128,0.0033258666594823206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,128,0.015698132912317912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,64,0.003197866678237915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,64,0.01548799971739451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,256,32,0.0032501332461833954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,256,32,0.015577600399653117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,3072,0.025033599138259886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,65536,0.13055893580118816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,65536,0.6190240224202473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,51200,0.10268692970275879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,2560,0.021580799420674642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,51200,0.4840138753255208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,16384,0.0356874664624532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,16384,0.16579732894897461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,1536,0.015041066209475198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,12288,0.028864000240961713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,12288,0.12770666281382242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,12288,0.08267947038014731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,768,0.009539199868837993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,10240,0.10829119682312012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,256,0.006557866434256236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,10240,0.07089599768320719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,8192,0.015948800245920818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,8192,0.08850560188293458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,128,0.005643733342488607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,7168,0.01578986644744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,7168,0.07680213451385498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,256,2048,0.018116267522176106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,6144,0.014270933469136557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,6144,0.06444373528162638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,5120,0.01163093348344167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,5120,0.051844267050425205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,5120,0.037196799119313555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,4096,0.010003200173377991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,65536,0.4165696144104004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,10240,0.025121066967646283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,3584,0.009612799684206644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,4096,0.030661332607269286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,3584,0.03478826681772868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,3072,0.008407466610272725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,3072,0.03039466738700867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,51200,0.329313055674235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,2560,0.007574399809042613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,2560,0.028113067150115967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,16384,0.10930133660634359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,2048,0.00673173318306605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,4096,0.03957653443018595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,2048,0.01734506686528524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,1536,0.0058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,1536,0.02271040081977844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,8192,0.058316799004872646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,1024,0.004819199939568837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,1024,0.01990506649017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,7168,0.050879999001820886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,768,0.004390400151411692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,768,0.018601600329081217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,6144,0.044437332948048906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,512,0.0038592000802357995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,2048,0.025388799111048382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,512,0.01758506695429484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,256,0.0033674667278925574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,256,0.016321067015329996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,3584,0.026547199487686156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,128,0.0031989333530267083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,128,0.01562879979610443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,3072,0.02315839926401774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,2560,0.020194133122762047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,64,0.0030570665995279947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,64,0.015219199657440185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,128,32,0.0031317333380381264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,128,32,0.015243732929229736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,1536,0.013857066631317139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,65536,0.11781333287556965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,51200,0.09328746795654297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,65536,0.616534423828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,51200,0.48076480229695634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,16384,0.033522133032480875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,1024,0.010999466975529988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,12288,0.02626240054766337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,10240,0.022716800371805825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,768,0.009081600109736125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,16384,0.16457279523213703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,12288,0.12689493497212728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,8192,0.013979732990264893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,10240,0.1076032002766927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,7168,0.012784000237782797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,6144,0.012486400206883748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,8192,0.08736000061035157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,5120,0.010940800110499065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,4096,0.009310932954152425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,6144,0.06351039807001749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,5120,0.05044480164845785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,3584,0.008826667070388794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,512,0.0074314668774604796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,3072,0.008056533336639405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,4096,0.03870826562245687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,3584,0.03394346634546916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,3072,0.030346665779749555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,2048,0.006358399987220764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,2560,0.027829333146413164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,1536,0.005358933409055074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,2048,0.02492799957593282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,1024,0.004589866598447164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,1536,0.022295467058817544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,768,0.004149333387613296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,256,0.006357333560784657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,1024,0.019322667519251505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,512,0.00367253323396047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,768,0.019056000312169395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,256,0.0031487998863061273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,128,0.0029130667448043824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,512,0.017511467138926186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,64,0.002794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,128,0.015286399920781454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,32,0.0028768000503381092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,64,0.015041066209475198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,32,0.014802133043607077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2305,128,128,0.005467733244101206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,65536,0.11622933546702068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,51200,0.09358932971954345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,12288,0.02564799984296163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,16384,0.034380801518758136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,10240,0.02189013361930847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,16384,0.1631285349527995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,12288,0.1253376007080078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,8192,0.01337386667728424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,10240,0.10564586321512859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,7168,0.012617599964141846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,6144,0.011294933160146077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,8192,0.08615573247273764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,7168,0.07433386643727621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,51200,0.48215465545654296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,5120,0.009102933605511983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,4096,0.008774399757385254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,6144,0.06252266565958658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,65536,0.6153461456298828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,3584,0.008570667107899983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,4096,0.038812800248463945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,5120,0.04927039941151937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,3072,0.007630933324495952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,2560,0.007051733136177063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,3584,0.03328746755917867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,2048,0.005836800237496694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,3072,0.030100266138712566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,7168,0.076036270459493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,1536,0.005027199784914652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,2048,0.02482666571935018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,768,0.003941333293914795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,512,0.0035797332723935447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,1536,0.022077866395314536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,768,0.017992534240086875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,256,0.003102933367093404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,512,0.016952532529830932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,128,0.0029311999678611755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,256,0.015719466408093772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,64,0.0027776000400384264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,128,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,32,0.002841600030660629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,64,0.01486186683177948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,32,0.014885333180427552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,64,256,0.015914666652679443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,2560,0.027483733495076497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,32,1024,0.004323199888070424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2305,32,1024,0.019512534141540527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2305,64,2560,0.007378133138020833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,16384,3.760016123453776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,16384,4.245555114746094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,12288,5.258695475260416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,16384,6.911386617024739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,12288,2.6666015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,12288,3.2958997090657554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,10240,2.3511029561360677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,10240,4.112173970540365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,10240,2.8120137532552083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,8192,1.8073738098144532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,8192,2.1224639892578123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,7168,1.6211402893066407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,8192,3.549494425455729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,51200,11.02956034342448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,7168,3.032044728597005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,7168,1.9374474843343097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,6144,2.523242696126302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,6144,1.3300426483154297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,5120,1.2570805867513022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,6144,1.5880767822265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,5120,1.3973227183024088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,5120,2.2705482482910155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,4096,1.0028458913167317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,51200,15.463062540690103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,3584,0.8777290980021158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,4096,1.207300313313802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,4096,1.8883455912272136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,3584,1.593947728474935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,3584,1.0172437032063804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,2560,0.61212584177653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,3072,0.8268288294474283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,3072,0.6894282658894857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,2560,0.7048650741577148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,2560,1.040339215596517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,3072,1.263860321044922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,2048,0.5005887985229492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,1536,0.37044054667154946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,1024,0.486845874786377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,1536,0.6299039840698242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,2048,0.6145322799682618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,1536,0.5241973241170247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,1024,0.28569812774658204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,512,0.18823893864949542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,512,0.24833280245463052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,768,0.24208319981892906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,768,0.35621760686238607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,768,0.3431338628133138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,1024,0.37936105728149416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,256,0.16011093457539877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,2048,0.8666922887166342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,128,0.1304458697636922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,256,0.24690346717834472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,256,0.14499732653299968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,64,0.1252234697341919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,32,0.11900266806284587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,128,0.20531306266784669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,32,0.12984000047047933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,65536,512,0.2656938711802165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,64,0.13180586496988933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,65536,51200,22.52828572591146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,65536,128,0.13315306504567465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,16384,3.0298133850097657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,16384,5.686790466308594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,16384,3.4801439921061195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,51200,8.85494384765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,12288,4.197056070963542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,65536,11.4311767578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,12288,2.237965901692708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,51200,10.967679850260417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,12288,2.602124786376953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,10240,1.8740042368570964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,8192,1.6190645853678387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,10240,3.485700225830078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,10240,2.1530420939127604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,8192,3.1272628784179686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,65536,17.9516357421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,7168,1.2119327545166017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,51200,18.19465535481771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,8192,1.7793258666992187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,7168,2.475494384765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,6144,1.077612813313802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,6144,2.0545674641927083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,6144,1.3329461415608725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,5120,1.0241664250691733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,7168,1.5192223866780599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,5120,1.671673583984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,4096,0.7040800094604492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,5120,1.0834325154622397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,3584,0.6207541147867839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,4096,0.908514149983724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,4096,1.3554122924804688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,3584,1.244109853108724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,3072,0.5869546890258789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,3584,0.7756863911946614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,3072,0.7082784016927083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,2560,0.5020074526468913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,3072,1.040446917215983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,2560,0.8633109410603842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,2048,0.40073493321736653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,2048,0.6748287836710612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,1536,0.34347947438557946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,1536,0.5215893427530924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,2048,0.5210794766743978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,1536,0.46321706771850585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,1024,0.364570681254069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,1024,0.24048959414164223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,768,0.2087637265523275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,1024,0.332804266611735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,512,0.2034752051035563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,768,0.2972426732381185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,512,0.15795092582702636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,256,0.13419307072957357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,256,0.13438186645507813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,65536,22.312174479166664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,128,0.10456639925638836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,256,0.18582933743794758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,64,0.0966048002243042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,128,0.11001173655192058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,32,0.09537279605865479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,64,0.1092362642288208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,51200,32,0.10720106760660808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,128,0.20720213254292807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,51200,768,0.2775360107421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,512,0.22842346827189125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,16384,1.6652351379394532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,51200,3.1367220560709637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,65536,4.093191528320313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,51200,3.4274922688802087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,65536,4.695235188802084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,16384,0.9714911778767904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,16384,1.0994208017985025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,12288,0.7579797108968098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,12288,1.2671754201253256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,51200,5.684086608886719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,51200,2560,0.5481695810953776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,12288,0.8433472315470377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,10240,0.6394869486490886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,10240,1.0715819040934245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,8192,0.4949941317240397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,8192,0.8112735748291016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,7168,0.44449173609415693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,8192,0.5369696299235026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,7168,0.5015690803527832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,65536,6.772164408365886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,7168,0.7994218826293945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,6144,0.4052480061848958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,5120,0.34223893483479817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,6144,0.4158410708109538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,6144,0.6511615753173828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,5120,0.5280405362447103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,4096,0.27027947107950845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,5120,0.39795945485432943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,4096,0.3144330660502116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,3584,0.36160214742024743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,3584,0.2553269386291504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,3584,0.2755530675252279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,3072,0.3011125246683756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,2560,0.18184213638305663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,3072,0.2219221274058024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,3072,0.236899201075236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,2560,0.2786474545796712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,2560,0.19619839986165363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,2048,0.2070869286855062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,1536,0.15688640276590984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,2048,0.1481642723083496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,2048,0.16119786898295085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,1536,0.1267082691192627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,1536,0.13600746790568036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,1024,0.1182207981745402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,768,0.08858346939086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,1024,0.08818453152974447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,1024,0.11088746388753254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,768,0.08253973325093587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,768,0.09381759961446126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,512,0.0694976011912028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,256,0.04404160181681315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,512,0.06245546738306681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,512,0.07736213207244873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,128,0.03696320056915283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,256,0.05465173323949178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,64,0.03327253262201945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,128,0.059794131914774576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,32,0.03410666783650716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,64,0.049017600218455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,32,0.049505066871643064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,16384,4096,0.4531221389770508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,10240,0.6459871927897136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,16384,256,0.0643402655919393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,16384,128,0.04782826503117879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,16384,0.7529514948527019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,16384,1.2239232381184897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,51200,2.4772982279459637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,16384,0.868663469950358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,65536,3.069701385498047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,12288,0.993725840250651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,51200,2.74212163289388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,12288,0.5915274937947591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,65536,3.611466725667318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,12288,0.6945664087931316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,51200,3.9132853190104164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,10240,0.7747392018636068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,10240,0.520749855041504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,8192,0.4474090576171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,8192,0.6094826380411784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,10240,0.5697909037272135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,8192,0.4523626645406087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,7168,0.36053012212117513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,7168,0.5524906794230143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,6144,0.31547199885050453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,6144,0.3390890757242838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,7168,0.4019306818644206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,65536,4.907643636067709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,6144,0.5128042538960774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,5120,0.3868991851806641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,5120,0.2669034639994303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,4096,0.24132159550984703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,5120,0.2820800145467123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,4096,0.3211008071899414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,3584,0.19019625981648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,3584,0.2125706672668457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,3584,0.3016117413838705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,4096,0.2324906667073568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,3072,0.16770559946695965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,3072,0.25263466835021975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,3072,0.19534826278686523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,2560,0.19404266675313314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,2560,0.14570453961690266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,2048,0.15694185892740886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,2048,0.120414932568868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,1536,0.1276853322982788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,1536,0.10093973477681477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,1536,0.11614080270131429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,2048,0.15086933771769206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,1024,0.07502079804738362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,768,0.07331093152364096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,1024,0.09250986576080322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,512,0.05140373309453329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,768,0.06353493531545004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,768,0.07849706808725992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,256,0.034745601812998454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,512,0.05336639881134033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,512,0.06410986582438151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,256,0.04301760196685791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,256,0.05431893269220987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,128,0.041221332550048825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,64,0.026361600557963057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,32,0.02741546630859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,32,0.04227519830067952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,1024,0.08643306891123453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,12288,128,0.02902719974517822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,128,0.04994026819864909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,12288,64,0.04252586762110393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,12288,2560,0.14968533515930177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,16384,1.0195562362670898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,65536,2.683327992757161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,16384,0.6636970520019532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,16384,0.6882634480794271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,51200,2.0217877705891927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,12288,0.7672373453776042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,51200,2.201654307047526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,65536,2.881128438313802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,12288,0.5207189242045085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,65536,4.186410776774088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,12288,0.5474271774291992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,10240,0.43789119720458985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,10240,0.6533450444539388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,10240,0.43833281199137375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,8192,0.35335893630981446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,7168,0.3000469207763672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,8192,0.5200768152872721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,51200,3.3578580220540366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,8192,0.35975786844889324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,7168,0.46298987070719405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,7168,0.3291839917500814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,6144,0.40882132848103836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,6144,0.268779722849528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,6144,0.25804053942362465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,4096,0.19555306434631348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,5120,0.22291839917500816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,4096,0.2771072069803874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,5120,0.23233599662780763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,3584,0.22400852839152016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,4096,0.20513493220011392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,3584,0.1632138729095459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,3584,0.16630293528238932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,3072,0.19069120089213054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,3072,0.13881492614746094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,3072,0.156441593170166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,2560,0.16390080451965333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,2560,0.12630826632181805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,2048,0.1319818655649821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,1536,0.09188160101572672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,1536,0.08776640097300212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,1024,0.07480106353759766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,1024,0.0713589350382487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,768,0.05914880037307739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,1024,0.06896213690439859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,512,0.043975468476613364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,768,0.06259093284606934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,512,0.04743359883626302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,512,0.052596267064412436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,256,0.029783467451731365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,256,0.039287467797597245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,256,0.0424949328104655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,128,0.02410986622174581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,128,0.035894401868184406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,128,0.03913173278172811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,64,0.02110613385836283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,5120,0.33256638844807945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,64,0.03620800177256266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,32,0.02248426675796509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,32,0.036374398072560626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,2560,0.13268373012542725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,2048,0.10738986333211262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,10240,768,0.05837546586990357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,10240,1536,0.10968106587727863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,10240,2048,0.10129706859588623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,65536,2.174055480957031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,51200,1.765383529663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,16384,0.7863168080647787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,51200,2.75340576171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,16384,0.5511456171671549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,16384,0.5917930603027344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,12288,0.5733376185099284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,12288,0.4575200080871582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,65536,3.4468874613444007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,12288,0.42502079010009763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,51200,1.8103936513264973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,10240,0.487063471476237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,10240,0.3927274703979492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,10240,0.3462954521179199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,8192,0.4195136070251465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,8192,0.3062133471171061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,7168,0.28711252212524413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,8192,0.3183210690816244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,7168,0.27450879414876306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,7168,0.3986720085144043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,6144,0.3227189381917318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,6144,0.22962133089701334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,5120,0.19894293149312336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,5120,0.196723206837972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,5120,0.266211207707723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,4096,0.16477440198262533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,4096,0.22870292663574218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,4096,0.15738666852315267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,3584,0.18217066129048665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,3584,0.14727039337158204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,3072,0.15738026301066083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,3072,0.12188479900360108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,3072,0.1338154633839925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,2560,0.13452693621317546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,2048,0.10592640240987142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,2560,0.10957760016123455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,2560,0.10934399763743083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,2048,0.09628799756368002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,2048,0.0954912026723226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,1536,0.08258986473083496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,1024,0.06140480041503906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,1536,0.07837440172831217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,768,0.04998613198598226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,1024,0.058823466300964355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,1024,0.059196801980336514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,768,0.050588798522949216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,512,0.037212800979614255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,768,0.05168426831563314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,512,0.043884801864624026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,512,0.04365013440450032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,256,0.025689599911371867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,128,0.01987839937210083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,256,0.036755200227101645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,128,0.032258133093516037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,64,0.01793173352877299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,8192,32,0.019242666165033975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,64,0.03237760066986084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,32,0.03285440007845561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,3584,0.14715946515401204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,65536,2.441248067220052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,8192,1536,0.07551679611206055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,256,0.03739413420359294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,128,0.033343998591105144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,16384,0.6804288228352864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,8192,6144,0.21129066149393716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,65536,1.9693621317545573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,51200,1.5297162373860678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,16384,0.48789119720458984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,12288,0.5327498753865559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,65536,2.1986666361490887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,65536,3.035813395182292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,51200,1.6662357330322266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,12288,0.4129013379414876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,12288,0.39856106440226235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,51200,2.26143798828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,10240,0.4739466667175293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,10240,0.321946652730306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,10240,0.3658528010050456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,8192,0.3936415990193685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,8192,0.2928138732910156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,7168,0.24956053098042807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,7168,0.33014186223347985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,7168,0.23077333768208824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,6144,0.276804256439209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,6144,0.21906240781148276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,5120,0.22571733792622886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,5120,0.18409813245137532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,5120,0.17736639976501464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,4096,0.17931626637776693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,4096,0.145524263381958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,4096,0.14053227106730143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,3584,0.16236054102579753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,16384,0.5058560053507487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,3584,0.1240447998046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,3072,0.13741226196289064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,3072,0.1182751973470052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,3072,0.10623679955800373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,2560,0.1146122694015503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,2560,0.09942186673482259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,2048,0.09295466740926107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,2560,0.09784533182779948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,2048,0.08281813462575277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,2048,0.0847370704015096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,1536,0.07182613213857016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,1536,0.06957013607025146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,1024,0.05162239869435629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,1024,0.05277226765950521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,1024,0.05461440086364746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,768,0.041509334246317545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,512,0.03196373383204142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,768,0.048529068628946945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,512,0.04023040135701497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,512,0.03938773473103841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,256,0.021943465868632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,256,0.032979200283686325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,256,0.03186986645062764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,128,0.01732053359349569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,128,0.029345067342122395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,64,0.015585066874821982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,64,0.029681066672007244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,7168,32,0.016285866498947144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,32,0.02951786716779073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,3584,0.1300693353017171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,1536,0.06806506315867106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,7168,768,0.04611306587855021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,128,0.03004266619682312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,8192,0.25248533884684243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,7168,6144,0.1925930658976237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,65536,1.730782953898112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,16384,0.5679541269938151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,16384,0.4372213363647461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,65536,1.8424043019612628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,65536,2.6223040262858075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,12288,0.33833068211873374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,16384,0.4475562731424968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,12288,0.47381013234456376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,51200,1.3894400278727213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,51200,2.0113780975341795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,51200,1.425542449951172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,12288,0.35977706909179685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,10240,0.28298559188842776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,8192,0.319863478342692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,10240,0.36836160024007164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,8192,0.2499786694844564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,7168,0.19559040069580078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,6144,0.23795733451843262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,7168,0.2105717341105143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,6144,0.17907412846883136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,6144,0.1780245304107666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,5120,0.1823861281077067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,5120,0.1552853266398112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,4096,0.15005119641621906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,4096,0.12004480361938477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,4096,0.12805333137512206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,3584,0.13316160043080646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,3584,0.11231466929117839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,3584,0.1060640017191569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,3072,0.11691306432088215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,3072,0.10323519706726074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,3072,0.09485119978586833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,2560,0.09607253074645997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,2560,0.08238399823506673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,2560,0.08761066595713297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,2048,0.07378559907277425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,2048,0.0698527971903483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,10240,0.29013334910074867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,1536,0.061459199587504065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,1536,0.05754026571909586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,1024,0.04470506509145101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,8192,0.26795520782470705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,7168,0.29173332850138345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,1024,0.04848320086797078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,768,0.0365941325823466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,512,0.028299733002980547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,768,0.04319146474202474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,768,0.04046826759974162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,512,0.03371306657791138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,256,0.020323199033737183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,256,0.02828906575838725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,128,0.01572266618410746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,256,0.030754133065541582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,128,0.02837973237037659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,128,0.02619413336118062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,64,0.013638400038083396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,32,0.013823999961217245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,64,0.027974400917689008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,5120,0.14583253860473633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,32,0.028282666206359865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,2048,0.0799605369567871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,6144,1536,0.06325120131174723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,6144,512,0.03669333457946777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,65536,1.5619402567545573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,65536,2.2210421244303387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,51200,1.2639060974121095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,16384,0.524948279062907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,51200,1.6807467142740886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,16384,0.4350005467732747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,12288,0.41373119354248045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,6144,1024,0.04538666804631551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,12288,0.3260863939921061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,10240,0.35332374572753905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,10240,0.2614357312520345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,8192,0.21285653114318848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,8192,0.2821770668029785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,7168,0.2223968029022217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,7168,0.20423146883646645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,6144,0.198689063390096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,7168,0.19540373484293622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,6144,0.1626442591349284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,6144,0.1715295950571696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,5120,0.16214399337768554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,5120,0.14240427017211915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,4096,0.13209706942240398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,4096,0.11555626392364501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,4096,0.11466986338297527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,3584,0.11489493052164715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,3584,0.10255573590596516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,65536,1.8246699015299481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,3072,0.1192629337310791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,3072,0.10762879848480225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,51200,1.4703722635904948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,2560,0.10211946964263915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,2560,0.09403733412424722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,10240,0.28071254094441733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,16384,0.4333183924357097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,2048,0.07104213237762451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,12288,0.3255743980407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,2048,0.07626026471455892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,2048,0.06538879871368408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,1536,0.05382293462753296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,1024,0.03970986604690552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,1536,0.05083093245824179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,1536,0.06255573431650797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,1024,0.04515626827875773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,768,0.036728533109029134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,1024,0.04036053419113159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,8192,0.21701119740804037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,512,0.025498666365941364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,768,0.03414933284123738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,768,0.03963306744893392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,256,0.018951465686162315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,512,0.03357760111490886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,256,0.029372799396514892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,128,0.013944533467292786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,128,0.01833066741625468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,128,0.026115200916926068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,64,0.012274133165677388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,64,0.025833600759506227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,5120,32,0.02621440092722575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,5120,0.13882239659627277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,3584,0.1002837340037028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,5120,32,0.012819199760754903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,65536,1.6657279968261718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,3072,0.08471360206604003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,65536,1.3269002278645834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,16384,0.3829312006632487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,2560,0.0767797311147054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,51200,1.2696245829264323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,16384,0.3839178721110026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,65536,1.5609493255615234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,51200,1.0533066431681315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,256,0.021319466829299926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,5120,512,0.02590293288230896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,12288,0.2874762535095215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,10240,0.2375413258870443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,10240,0.22274239857991537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,10240,0.23772907257080078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,8192,0.19325226147969563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,8192,0.18590933481852215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,7168,0.17070080439249674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,8192,0.1883797327677409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,7168,0.16512640317281085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,6144,0.14733227094014484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,6144,0.13658666610717773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,5120,0.12435946464538575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,5120,0.11583573023478191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,4096,0.10117866992950439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,4096,0.09642879962921143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,12288,0.32616214752197265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,3584,0.08739946683247884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,3584,0.08569173018137613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,16384,0.35568211873372396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,3072,0.07645973364512125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,51200,1.187058130900065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,12288,0.268454392751058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,3072,0.0880394697189331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,3072,0.07240107059478759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,2560,0.07048319975535075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,2560,0.06683733463287353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,2048,0.05246293147404989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,2048,0.056927998860677086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,7168,0.1650602658589681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,1536,0.040591998895009355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,6144,0.13860267003377277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,1536,0.046538666884104414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,1024,0.029363199075063066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,1024,0.036363732814788816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,5120,0.11944639682769775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,768,0.023772799968719484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,1024,0.03425813515981038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,768,0.027949867645899455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,768,0.031616000334421794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,512,0.018427733580271402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,4096,0.09951786994934082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,512,0.02787733276685079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,256,0.012712533275286356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,256,0.023845332860946655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,128,0.009796266754468281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,128,0.024331732590993246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,128,0.015642666816711427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,64,0.010419199864069622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,64,0.024052266279856363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,4096,32,0.010920533537864685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,4096,32,0.024541866779327393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,3584,0.08611839612325033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,65536,1.4102240244547526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,65536,1.2325653076171874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,2560,0.06515520016352336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,2048,0.05462506612141928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,1536,0.041902931531270345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,16384,0.3332693417867025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,51200,1.1312299092610678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,16384,0.3180757204691569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,256,0.01833066741625468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,51200,0.9672320048014322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,12288,0.28455467224121095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,12288,0.24367039998372397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,10240,0.21332267125447593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,10240,0.17766079902648926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,8192,0.17204052607218426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,10240,0.22538453737894693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,8192,0.16643199920654297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,7168,0.15387733777364093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,7168,0.1460096041361491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,6144,0.13028480211893717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,6144,0.12632213433583578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,4096,512,0.022377600272496544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,6144,0.11235199769337971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,5120,0.10703573226928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,5120,0.10734826723734539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,5120,0.09409173329671225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,4096,0.08876907030741374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,4096,0.08921706676483154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,4096,0.07731520334879557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,3584,0.07675413290659586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,3584,0.08053226470947265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,3584,0.06879893143971762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,3072,0.06782720088958741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,8192,0.15671253204345703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,3072,0.060821334520975746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,2560,0.057076267401377355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,2560,0.06170453230539957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,2560,0.05347520112991333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,2048,0.04687360127766927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,2048,0.052692266305287686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,2048,0.04572906494140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,1536,0.03638720115025838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,1536,0.04247893492380778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,1536,0.038235731919606525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,1024,0.025887999931971235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,1024,0.03454826672871907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,1024,0.030078933636347456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,768,0.021552000443140665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,768,0.029600000381469725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,512,0.01650879979133606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,512,0.026029866933822633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,16384,0.27312211990356444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,256,0.0116757333278656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,65536,1.0838592529296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,256,0.022551467021306358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,128,0.0090037335952123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,51200,0.8819583892822266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,12288,0.21252053578694663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,128,0.024200532833735147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,64,0.010181333621342976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,64,0.024268800020217897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3584,32,0.01051200032234192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,3072,0.07175573507944742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3584,32,0.023693867524464927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,7168,0.1284511963526408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,65536,1.192510986328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,256,0.022291199366251627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,65536,1.3944661458333334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,51200,1.0291178385416666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,16384,0.34548905690511067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,51200,0.947111447652181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,16384,0.3316885312398275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,768,0.027141332626342773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,512,0.021844265858332317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,12288,0.2576042652130127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,12288,0.237936004002889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,10240,0.21106346448262533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,12288,0.23205439249674478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,10240,0.20009279251098633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3584,128,0.016823466618855795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,8192,0.16384746233622233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,10240,0.19566826820373534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,8192,0.1633962631225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,7168,0.14311680793762208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,7168,0.14911999702453613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,6144,0.12404053211212158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,6144,0.12581973075866698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,5120,0.10452160040537517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,5120,0.10654079914093018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,5120,0.09831466674804687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,4096,0.08460266590118408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,4096,0.08823893070220948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,65536,1.254644266764323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,3584,0.08072746594746907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,3584,0.0800874630610148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,3584,0.0792138655980428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,3072,0.07006399631500244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,3072,0.06956266562143962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,2560,0.054308267434438076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,2560,0.059628800551096595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,2560,0.05293973286946615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,2048,0.04475626548131307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,51200,0.9701962788899741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,2048,0.04970133304595947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,1536,0.034371201197306314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,2048,0.04591253201166789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,1536,0.040321067969004316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,16384,0.30000212987263997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,1024,0.024046933650970458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,1024,0.032763733466466265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,768,0.01946773330370585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,768,0.02908373276392619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,8192,0.1477567990620931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,7168,0.13594667116800946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,512,0.014596266547838846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,512,0.026392533381779985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,256,0.010971732934316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,6144,0.1150058666865031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,256,0.022452267011006673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,128,0.008518399794896443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,128,0.02029119928677877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,4096,0.08115306695302328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,64,0.007751466830571492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,64,0.020466132958730062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,3072,32,0.009512533744176228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,3072,32,0.020470400651295982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,3072,0.05896533330281576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,1536,0.033845333258310954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,65536,1.0370709101359048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,65536,1.0600213368733724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,768,0.0212501327196757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,512,0.01709653337796529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,51200,0.8877887725830078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,51200,0.8179360071818034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,51200,0.8368618647257486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,256,0.013941333691279093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,16384,0.24999359448750816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,12288,0.18984853426615397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,128,0.012075733145078022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,12288,0.21105173428853355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,10240,0.16203306516011556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,3072,1024,0.027845333019892376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,10240,0.1694464047749837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,10240,0.1771242618560791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,8192,0.131440003712972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,7168,0.11327359676361085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,8192,0.14583360354105632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,8192,0.1345311959584554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,7168,0.13305386702219646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,6144,0.09935466448465982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,7168,0.11993920008341472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,6144,0.11021440029144287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,6144,0.10688426494598388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,5120,0.08365333080291748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,5120,0.09434666633605956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,4096,0.06731093724568685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,4096,0.08011519908905029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,3584,0.05988906621932984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,3584,0.07137706279754638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,3072,0.05164586702982584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,3072,0.06299413442611694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,3072,0.05279786586761474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,2560,0.04353919823964437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,16384,0.29420693715413415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,2560,0.05354666709899902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,2048,0.035571201642354326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,2048,0.04466559886932373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,3584,0.06338346799214681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,65536,1.0576138814290366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,1536,0.03143786589304606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,1536,0.04120106697082519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,1536,0.031568000713984176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,16384,0.26139839490254724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,1024,0.021970132986704506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,1024,0.02550293405850728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,1024,0.03041599988937378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,12288,0.19351253509521485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,768,0.018186666568120322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,512,0.013920000195503235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,768,0.02604159911473592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,768,0.01939199964205424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,256,0.009242666761080424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,512,0.015879467129707336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,512,0.024106667439142863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,128,0.007604266703128815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,256,0.013364266355832419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,256,0.022652800877889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,64,0.008366933465003968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,128,0.021949867407480873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2560,32,0.009113599856694538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,64,0.021918932596842446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2560,32,0.022038400173187256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,5120,0.08702293237050375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,4096,0.07264959812164307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,65536,0.7397162755330403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,51200,0.5932533264160156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,65536,0.931336530049642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,2560,0.04752426544825236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,16384,0.18877654075622557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,2048,0.03874986569086711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,16384,0.24565866788228355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,12288,0.14459093411763507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,51200,0.7566303888956706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,12288,0.18717652956644695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,10240,0.1193013350168864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,10240,0.15746134122212727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,8192,0.09572800000508627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,8192,0.12877333164215088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,7168,0.0840554634730021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,7168,0.1147722641626994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2560,128,0.01151146690050761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,6144,0.0722218672434489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,6144,0.09935999711354573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,12288,0.18214826583862304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,5120,0.06267626682917277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,5120,0.08474240303039551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,65536,0.9863871892293294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,4096,0.05473493337631226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,4096,0.07520000139872232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,16384,0.2471989313761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,3584,0.044701866308848065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,3584,0.061698134740193686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,10240,0.15675519307454427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,3072,0.038429868221282956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,3072,0.053743998209635414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,6144,0.09682559967041016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,2560,0.03232959906260173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,2560,0.045551999409993486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,51200,0.7850346883138021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,2048,0.031056000789006548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,2048,0.043270401159922284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,8192,0.11929173469543457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,1536,0.02182719906171163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,7168,0.10922239621480305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,1536,0.03306986689567566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,1024,0.016310399770736693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,1024,0.02757973273595174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,3072,0.04566826820373535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,768,0.013478400309880576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,768,0.02411306699117025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,4096,0.0627455989519755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,512,0.010488532980283101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,1536,0.025983999172846477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,512,0.021898667017618813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,256,0.007795199751853943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,256,0.019683200120925903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,5120,0.07631253401438395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,128,0.006554666658242543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,2560,0.040608000755310056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,128,0.018461867173512777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,64,0.005798399945100148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,64,0.018317866325378417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,2048,32,0.006067200005054474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,2048,32,0.018314667542775474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,3584,0.05331519842147827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,1024,0.02123946746190389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,65536,0.6892970403035481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,2048,0.03314773241678874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,65536,0.8725439707438151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,51200,0.5771999994913737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,16384,0.1886112054189046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,51200,0.6875263849894206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,16384,0.2501728057861328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,12288,0.1263423999150594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,12288,0.18068159421284993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,65536,0.7059210459391276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,256,0.01172266701857249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,51200,0.5720469156901042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,10240,0.10442986488342285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,10240,0.1488800048828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,128,0.01034346620241801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,16384,0.1718143939971924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,8192,0.08161280155181885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,8192,0.12154560089111328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,7168,0.07184106508890788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,8192,0.08919573624928792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,6144,0.062226132551829016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,7168,0.10791573524475098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,512,0.014101333419481912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,6144,0.09446293512980143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,5120,0.05205440123875936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,5120,0.07983253002166749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,5120,0.058124800523122154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,12288,0.1372970740000407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,4096,0.042854400475819905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,4096,0.06475520133972168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,2048,768,0.016950400670369466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,4096,0.04864213466644287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,3072,0.03368213176727295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,3584,0.057017600536346434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,3072,0.05020800034205118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,2560,0.028830933570861816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,2560,0.04229226509730021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,3584,0.03816639979680379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,2048,0.024315732717514037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,2048,0.036673065026601157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,10240,0.11359679698944092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,1536,0.019478400548299156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,1536,0.031113600730895995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,1024,0.012714667121569314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,7168,0.07859093348185221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,1024,0.025253333648045856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,768,0.010161067048708599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,6144,0.07122773329416911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,768,0.022180267175038657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,512,0.008017066617806752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,512,0.020594133933385213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,3072,0.03650986750920614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,3584,0.041917868455251056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,256,0.006332799792289734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,256,0.018131200472513834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,128,0.0056618665655454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,2560,0.03252266645431519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,128,0.017502933740615845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,64,0.005318399767080943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,64,0.01744106610616048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1536,32,0.005486933390299479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1536,32,0.017911465962727864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,65536,0.4080704053243001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,2048,0.02665173411369324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,65536,0.7743882497151693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,65536,0.6378005345662434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,51200,0.3192469278971354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,1536,0.021270400285720824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,16384,0.10522346496582032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,768,0.014109866817792258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,51200,0.6033695856730144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,16384,0.19711146354675294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,512,0.01162559986114502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,12288,0.07621866861979167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,12288,0.1527882734934489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,10240,0.06403520107269287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,256,0.009543466567993163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,8192,0.05068906545639038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,128,0.008414933085441589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,8192,0.10589013099670411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,7168,0.044949332873026535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,7168,0.09419413407643637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1536,1024,0.016951467593510947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,6144,0.0395146648089091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,6144,0.08165866533915202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,10240,0.12817813555399576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,5120,0.033462401231129965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,5120,0.06794559955596924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,51200,0.5069269180297852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,4096,0.02726080020268758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,16384,0.16235413551330566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,4096,0.055181864897410074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,3584,0.025010132789611818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,12288,0.124836270014445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,3584,0.047518932819366456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,3072,0.022270933787027995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,3584,0.03845440149307251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,10240,0.10470933119455975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,2560,0.01812160015106201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,3072,0.04157653252283732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,2560,0.03586879968643188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,2048,0.015314132968584696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,8192,0.0836949348449707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,2048,0.03118720054626465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,1536,0.012436266740163167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,1536,0.026789333422978717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,1536,0.01922986706097921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,1024,0.009418666362762451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,7168,0.07360640366872152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,1024,0.022810665766398112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,768,0.008076799909273784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,768,0.02105493346850077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,6144,0.06368639866511026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,5120,0.0526581327120463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,512,0.006727466483910878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,512,0.020150399208068846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,4096,0.043562666575113936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,256,0.0053962667783101406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,256,0.0176362673441569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,3072,0.03346879879633586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,128,0.004655999938646952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,2560,0.028511999050776164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,128,0.016909867525100708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,64,0.00425600012143453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,64,0.016810667514801026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,1024,32,0.004531200230121613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,1024,32,0.016953599452972413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,1024,0.015128533045450846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,65536,0.3019808133443197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,768,0.012630400061607362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,65536,0.501692803700765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,51200,0.23918827374776205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,16384,0.07856853008270263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,51200,0.554196294148763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,2048,0.02323626677195231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,12288,0.06017706791559855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,12288,0.14671893119812013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,65536,0.7026250839233399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,10240,0.05341013272603353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,256,0.008528000116348267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,10240,0.12471466859181721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,16384,0.18600319226582845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,8192,0.10126826763153077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,128,0.007577600081761678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,1024,512,0.010583466291427613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,7168,0.03597653309504191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,7168,0.09067306518554688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,6144,0.031591467062632245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,6144,0.0782965342203776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,6144,0.052517334620157875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,5120,0.029601067304611206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,5120,0.06435946623484293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,8192,0.039979732036590575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,4096,0.02469866673151652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,4096,0.050545068581899014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,4096,0.03666666746139526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,3584,0.02062186598777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,51200,0.3930624008178711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,3584,0.043484799067179364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,16384,0.13244160016377765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,3072,0.020142932732899986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,3072,0.038372266292572024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,3072,0.028016000986099243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,2560,0.015511467059453329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,2560,0.032918399572372435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,2560,0.024001065889994302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,2048,0.013270399967829385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,2048,0.027909332513809205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,1536,0.010936533411343891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,1536,0.025042132536570234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,10240,0.08370986779530844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,1024,0.007975466549396515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,1024,0.02199466625849406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,8192,0.06837759812672933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,768,0.006945066650708516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,12288,0.10155306657155354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,768,0.020087466637293497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,7168,0.060602664947509766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,768,0.01111893355846405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,512,0.005784533421198527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,256,0.004680533210436503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,512,0.018902399142583213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,256,0.01707520087560018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,128,0.004142933338880539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,128,0.01627626617749532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,5120,0.04427413145701091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,64,0.0038133333126703895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,64,0.01621119976043701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,768,32,0.00401706670721372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,768,32,0.01597546637058258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,65536,0.2220693270365397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,3584,0.031753599643707275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,2048,0.0204693337281545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,65536,0.6707967758178711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,1536,0.0165802667538325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,51200,0.17228479385375978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,16384,0.0566271980603536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,16384,0.1765343983968099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,51200,0.525652281443278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,12288,0.04376320044199626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,1024,0.012935466567675271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,12288,0.1350144068400065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,10240,0.037137067317962645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,10240,0.11457599798838299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,512,0.009160533547401428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,256,0.007495466868082683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,8192,0.03050559957822164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,8192,0.0949461301167806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,768,128,0.006647466619809468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,7168,0.026324266195297243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,7168,0.08321066697438559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,6144,0.022550400098164877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,65536,0.504916254679362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,6144,0.07115946610768636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,6144,0.05272639989852905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,5120,0.01994133392969767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,51200,0.3917514801025391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,16384,0.13229653040568035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,5120,0.0576533317565918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,4096,0.016088533401489257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,4096,0.04554346799850464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,3584,0.014595199624697366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,3584,0.03948266506195068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,12288,0.09969493548075357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,3072,0.013125333189964294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,3072,0.03494400183359782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,10240,0.08386133511861166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,2560,0.011425066987673442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,2560,0.031165866057078044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,8192,0.06968106428782145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,2048,0.00997226635615031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,2048,0.027616000175476073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,7168,0.06258560021718343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,1536,0.008301866551240284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,1536,0.023820799589157105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,1536,0.016075733304023742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,1024,0.006760533154010773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,1024,0.020802134275436403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,768,0.005980800092220307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,768,0.01964799960454305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,5120,0.04505279858907064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,4096,0.036762666702270505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,512,0.005155199766159057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,512,0.018339200814565023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,256,0.004351999859015147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,256,0.016727467377980552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,3584,0.03232746720314026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,128,0.0038218667109807336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,3072,0.027617067098617554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,128,0.016425599654515587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,64,0.0036757332583268487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,64,0.016379732886950174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,512,32,0.003737599899371465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,512,32,0.016004266341527303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,65536,0.1339039961496989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,2560,0.023890133698781332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,2048,0.019883733987808228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,51200,0.10830612977345784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,65536,0.6295679728190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,1024,0.012346667051315308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,768,0.010609066486358643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,16384,0.04090346495310466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,16384,0.17206400235493977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,12288,0.03377279837926229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,51200,0.49268798828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,12288,0.1338314692179362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,10240,0.02994133234024048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,10240,0.11355626583099365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,512,0.008644266923268636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,8192,0.025095466772715253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,256,0.006985599795977275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,8192,0.09226773579915365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,512,128,0.005985066791375478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,7168,0.020787199338277183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,7168,0.07859946886698405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,6144,0.017756799856821694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,6144,0.06650240023930867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,65536,0.4470645268758138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,5120,0.015500799814860026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,12288,0.088100266456604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,5120,0.05322133302688599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,4096,0.011865599950154623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,4096,0.04100586573282878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,16384,0.11602986653645833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,3584,0.010215466221173603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,3584,0.035575465361277266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,51200,0.34629761377970375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,3072,0.009275733431180318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,3072,0.031640533606211343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,10240,0.07613226572672525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,2560,0.007948799928029378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,2560,0.028193066517512005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,6144,0.0466261347134908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,2048,0.007284266750017802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,2048,0.026658133665720625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,7168,0.05370133320490519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,1536,0.006029866635799408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,8192,0.06113813320795695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,1536,0.022776534159978233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,1024,0.005155199766159057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,1024,0.020165334145228066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,4096,0.031996800502141313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,768,0.004633600016434988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,3072,0.02414720058441162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,768,0.018540799617767334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,512,0.004075733323891958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,5120,0.03949546813964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,512,0.017218132813771568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,256,0.0035455999275048576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,256,0.016116266449292503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,3584,0.027505065997441607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,128,0.003368533402681351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,2560,0.021421867609024047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,1536,0.014379733800888061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,128,0.01550933321317037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,64,0.0032501332461833954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,64,0.015076266725858054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,256,32,0.0032032000521818793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,256,32,0.0150218665599823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,1024,0.011180800199508668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,65536,0.12508800029754638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,2048,0.017650133371353148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,51200,0.0988266626993815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,65536,0.6195210774739583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,51200,0.486953608194987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,16384,0.03694506486256917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,51200,0.3130037307739258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,16384,0.10370346705118816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,12288,0.0290613333384196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,12288,0.1278879960378011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,16384,0.1684853394826253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,10240,0.025592533747355144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,10240,0.1089898665746053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,8192,0.02104319930076599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,8192,0.08842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,65536,0.40025920867919923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,7168,0.014432000120480857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,256,0.006438399851322174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,7168,0.07698346773783366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,6144,0.012612266341845193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,6144,0.06364266475041708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,128,0.005845333139101664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,512,0.008011733492215473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,5120,0.01115733285744985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,6144,0.04290879964828491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,5120,0.05082133213678995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,4096,0.009327999750773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,256,768,0.009679999947547913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,3584,0.008638933300971985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,4096,0.03955733378728231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,3584,0.0342741330464681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,3072,0.00795839975277583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,12288,0.07973546981811523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,3072,0.02222826679547628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,2560,0.00709440012772878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,2560,0.028585600852966308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,5120,0.03489280144373576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,2048,0.0063274666666984555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,10240,0.06669546763102213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,2048,0.026200532913208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,3584,0.025575466950734455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,1536,0.005564799904823304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,1536,0.013223466277122498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,1536,0.02270080049832662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,3072,0.030019199848175047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,1024,0.0047189335028330484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,768,0.004226133227348328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,1024,0.019717333714167277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,768,0.018547199169794717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,512,0.0038122666378815973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,8192,0.05430399974187215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,512,0.01742186745007833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,256,0.003433600068092346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,512,0.007517866790294647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,256,0.016012799739837647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,128,0.0031445334355036415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,128,0.015581867098808289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,7168,0.04926186800003052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,64,0.003010133405526479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,64,0.015056000153223673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,128,32,0.0029653333127498626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,128,32,0.015009066462516785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,4096,0.029922133684158324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,65536,0.11584426561991375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,768,0.009108266234397889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,51200,0.09148480097452799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,16384,0.03357119957605998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,2560,0.01934400002161662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,12288,0.02606613238652547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,16384,0.16427839597066243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,51200,0.4802709261576335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,65536,0.6132437388102214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,10240,0.022932267189025878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,8192,0.013424000144004822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,12288,0.1265674670537313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,10240,0.10753706296284993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,8192,0.08682666619618734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,6144,0.011548800269762675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,7168,0.07572906812032064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,5120,0.010029866298039754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,6144,0.0633130669593811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,4096,0.008558932940165203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,3584,0.007918933530648549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,5120,0.04977173407872518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,4096,0.03839146693547567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,3072,0.007243733108043671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,3584,0.03327786723772685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,2560,0.00639466643333435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,3072,0.031167999903361006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,2048,0.005625600119431814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,1536,0.0050122668345769245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,2048,0.02566186587015788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,1536,0.022243199745814006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,1024,0.004215466479460398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,2048,0.01648853321870168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,1024,0.01958079934120178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,768,0.0038474666575590765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,512,0.003551999976237615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,768,0.017962666352589925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,256,0.003143466760714849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,512,0.017619200547536216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,128,0.0029450667401154833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,256,0.015904000401496886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,64,0.002746666719516118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,128,0.015159466862678527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,32,0.0028064000109831494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,64,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,32,0.014633599917093912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,65536,0.11207146644592285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,256,0.0061930666367212926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,51200,0.08859732945760092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,16384,0.03157973289489746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,128,0.005478399991989136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,64,7168,0.013362133502960205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,12288,0.024962133169174193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,10240,0.021874133745829263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,16384,0.16303680737813314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,8192,0.011718400319417318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,12288,0.1252010663350423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,7168,0.010819199681282043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,65536,0.6106773376464844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,10240,0.10556906859079998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,6144,0.009635200103123982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,8192,0.08539626598358155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,51200,0.48055038452148435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,5120,0.008884267012278239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,7168,0.07483413219451904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,4096,0.0077354664603869125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,6144,0.06207893292109171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,3584,0.007319466769695282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,64,2560,0.027559467156728107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,3072,0.006985599795977275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,4096,0.037435734272003175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,2560,0.006270933151245117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,2048,0.00553599993387858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,3072,0.02970133423805237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,2560,0.0271999994913737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,1536,0.0048767998814582825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,1024,0.0042346666256586705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,2048,0.02467199961344401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,1536,0.022388267517089843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,768,0.00388373335202535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,1024,0.019486933946609497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,512,0.003504000107447306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,768,0.0184170663356781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,256,0.0031637333333492277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,512,0.017522132396697997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,128,0.0029098667204380036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,256,0.015761066476504007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,128,0.015309866269429526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,32,0.002792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,64,0.01471573313077291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,32,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2304,128,1024,0.010703999797503154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,5120,0.049131735165913897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2304,32,3584,0.033155200878779094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2304,32,64,0.0027935999135176343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,16384,3.3540982564290367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,12288,2.5096107482910157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,16384,4.436081949869791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,12288,4.362504577636718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,16384,5.979444376627604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,10240,1.993781280517578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,12288,3.238795725504557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,10240,3.8606783548990884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,10240,2.7587626139322916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,8192,2.98415044148763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,8192,1.687660853068034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,8192,2.028273010253906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,7168,1.4264821370442708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,51200,10.196586100260417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,7168,1.891756820678711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,6144,1.1628618876139323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,6144,1.5278250376383462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,6144,2.266124725341797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,5120,1.08721923828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,7168,2.58572260538737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,5120,2.178736114501953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,4096,0.7957674662272136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,5120,1.3369728088378907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,4096,1.5588427225748698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,4096,1.206063969930013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,3584,1.2369845072428385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,3584,0.9964362462361654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,3584,0.8885738372802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,3072,0.6894357045491536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,3072,1.4227274576822917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,3072,0.9048949559529623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,2560,0.6372842788696289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,2560,0.9914645512898763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,2560,0.8631989161173502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,2048,0.4633791923522949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,2048,0.8876725514729819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,1536,0.3584522565205892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,2048,0.6295093536376953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,1536,0.7553173065185547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,1024,0.2619232018788656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,1536,0.5463584264119465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,1024,0.4792917251586914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,1024,0.35520000457763673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,768,0.21075092951456703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,768,0.3850197474161784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,768,0.3282848040262858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,512,0.2665781339009603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,512,0.16068587303161622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,512,0.2522805372873942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,256,0.17046292622884113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,128,0.12226026852925617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,256,0.1432437260945638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,128,0.12731626828511555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,64,0.11300053596496581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,64,0.12431893348693848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,32,0.11097066402435303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,128,0.26070613861083985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,65536,32,0.12008319695790608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,256,0.2569514592488607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,65536,51200,19.625634765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,65536,51200,20.070304361979165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,16384,2.600108846028646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,16384,4.892030843098959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,16384,3.470163218180338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,51200,7.977977498372395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,12288,3.7658495585123695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,65536,10.79700927734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,12288,2.154222869873047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,10240,1.6568618774414063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,10240,3.0410219828287763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,10240,2.2967508951822917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,8192,1.6010548909505207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,8192,2.5106847127278646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,51200,16.52524617513021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,8192,1.7133951822916669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,7168,1.1906378428141275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,7168,1.455395253499349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,7168,2.2704437255859373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,6144,1.859147771199544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,6144,0.9919616063435873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,65536,19.7935791015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,5120,0.8246677398681641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,6144,1.2189236958821614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,5120,1.577667236328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,4096,0.7170080184936524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,4096,0.8380032221476237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,4096,1.2269696553548177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,65536,20.92884318033854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,3584,0.5882538477579753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,12288,2.494647471110026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,3584,1.1037770589192708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,3584,0.7239786783854167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,3072,0.9176479975382487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,3072,0.635963757832845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,2560,0.433513609568278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,2560,0.5413951873779297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,3072,0.5271242777506511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,2560,0.7582773208618164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,2048,0.3723402659098307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,2048,0.6476245244344075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,2048,0.46685333251953126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,1536,0.29485867818196615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,1536,0.47666133244832354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,1536,0.3983722686767578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,1024,0.33024959564208983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,51200,14.259945678710938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,1024,0.2257685343424479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,768,0.2508000055948893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,1024,0.2974368095397949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,512,0.1918453375498454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,768,0.25698773066202796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,256,0.13314666748046874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,256,0.11613439718882243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,128,0.10192639827728271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,512,0.2124085267384847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,256,0.19080533981323242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,128,0.1058463970820109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,64,0.09202773571014404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,128,0.15975786844889323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,64,0.10446080366770427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,51200,32,0.09045759836832681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,32,0.10107093652089436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,768,0.18867519696553547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,51200,512,0.156385072072347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,16384,1.5408191680908203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,51200,2.956764729817708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,65536,3.7163754781087244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,16384,0.9054677327473959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,51200,5120,0.975104014078776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,16384,1.0857332865397136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,12288,0.6812053044637044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,51200,3.559355672200521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,51200,5.006559753417969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,12288,1.1877450307210287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,65536,5.364615376790365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,10240,0.5740959803263347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,10240,1.0220917383829753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,10240,0.669971211751302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,8192,0.45912853876749676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,8192,0.7591221491495769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,7168,0.42826560338338215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,65536,6.113270568847656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,7168,0.4627669334411621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,7168,0.7248405456542969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,6144,0.346450138092041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,6144,0.5944629033406575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,6144,0.3955434799194336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,5120,0.31974614461263023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,5120,0.5152736028035482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,5120,0.3363029479980469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,4096,0.40403092702229815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,4096,0.24838719367980958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,4096,0.28178879419962566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,3584,0.3644895871480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,8192,0.5166463851928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,3072,0.29730987548828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,3072,0.19701439539591473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,3072,0.2503488063812256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,2560,0.17352533340454102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,2560,0.23173333803812662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,2048,0.14451947212219238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,2048,0.15190186500549316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,1536,0.15976959864298504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,1536,0.10780906677246094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,1024,0.1029205322265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,1536,0.14200746218363444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,1024,0.08295679887135823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,1024,0.10765653451283771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,768,0.08247786362965902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,768,0.07608426411946614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,768,0.08580693403879801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,512,0.06447039842605591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,512,0.059544531504313145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,3584,0.22044587135314941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,512,0.07495466868082681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,256,0.04255146582921346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,128,0.035913598537445066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,256,0.06284906864166259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,256,0.05072426795959473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,64,0.03177493413289388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,128,0.04490773280461629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,128,0.05770026842753092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,32,0.031931734085083006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,64,0.044869331518809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,16384,32,0.045186134179433186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,2560,0.1892533302307129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,16384,2048,0.212610133488973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,51200,2.206037394205729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,51200,2.6333824157714845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,65536,2.782884216308594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,51200,3.616744486490885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,16384,1.1660884857177733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,65536,3.5749610900878905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,16384,0.6741504033406576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,16384,0.7850229263305664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,12288,0.7652095794677735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,12288,0.5201216061909993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,65536,4.461382548014322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,12288,0.6030421574910482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,10240,0.44613332748413087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,10240,0.7122880299886067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,12288,0.8263519922892252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,10240,0.5161909421284994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,8192,0.5460479736328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,8192,0.40642560323079424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,8192,0.362994130452474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,7168,0.3235562642415365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,7168,0.48015572230021164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,6144,0.4317920049031575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,6144,0.290880012512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,5120,0.2234656016031901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,6144,0.3097066561381022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,5120,0.27215360005696615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,4096,0.29392960866292317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,4096,0.19412479400634766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,3584,0.17175679206848143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,4096,0.21385706265767418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,3584,0.23527253468831383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,3584,0.2116096019744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,3072,0.14574186007181805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,3072,0.16498559315999348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,3072,0.20746347109476723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,2560,0.13813865979512532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,2048,0.1401685396830241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,2560,0.1458677291870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,2048,0.10953386624654134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,1536,0.10874133110046387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,1536,0.08911253611246744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,2048,0.1257525364557902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,1536,0.0977567990620931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,1024,0.07795946598052979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,768,0.06307520071665446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,1024,0.06814080079396566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,1024,0.07775466442108155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,512,0.04772693316141764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,768,0.06757013003031412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,512,0.049114668369293214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,256,0.03328319986661275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,512,0.055937067667643225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,256,0.04164053201675415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,256,0.04667199850082397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,5120,0.3258687973022461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,128,0.027218133211135864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,128,0.03669120073318481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,64,0.024817067384719848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,128,0.044106666247049967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,32,0.026417066653569538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,64,0.037351465225219725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,32,0.037529599666595456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,12288,2560,0.1729109287261963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,12288,768,0.05885866483052572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,16384,3584,0.24044906298319496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,65536,2.448199462890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,51200,1.850647481282552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,65536,2.895379130045573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,51200,2.974924723307292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,51200,2.25686518351237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,16384,0.9427765528361002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,16384,0.5902602513631184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,16384,0.6606538772583008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,65536,3.8537633260091146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,12288,0.6768341064453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,12288,0.4579210599263509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,10240,0.38354132970174154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,8192,0.3175637245178223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,12288,0.5339167912801106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,10240,0.46344534556070965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,8192,0.46100053787231443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,8192,0.33983465830485027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,7168,0.2704597473144531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,7168,0.3972298622131348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,6144,0.24923626581827799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,7168,0.31783361434936525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,6144,0.3685738563537598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,5120,0.2737194697062174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,4096,0.23116799990336098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,4096,0.17627733548482258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,4096,0.2072533289591471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,3584,0.20824640591939292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,3584,0.1487167994181315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,3072,0.12743360201517742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,3072,0.1790112018585205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,2560,0.15165333747863768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,3072,0.14356692632039386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,2560,0.11777493158976238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,10240,0.5863402684529622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,2048,0.1247594674428304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,2048,0.09445652961730958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,2048,0.1093066692352295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,1536,0.0919274648030599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,6144,0.27276585896809896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,1536,0.07699413299560547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,1536,0.08292693297068278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,12288,7168,0.3573802630106608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,5120,0.2202997366587321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,1024,0.06100586652755737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,768,0.051275734106699625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,5120,0.23087786038716634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,768,0.0513322671254476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,512,0.03916586637496948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,768,0.05889279842376709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,256,0.026892799139022826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,512,0.04818559885025024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,512,0.04282879829406738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,256,0.03611413240432739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,256,0.0401962677637736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,128,0.03319680094718933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,64,0.019857066869735717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,128,0.03858986695607503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,32,0.020547199249267577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,64,0.03377600113550822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,10240,32,0.03345599969228109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,3584,0.1734666665395101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,2560,0.12437973022460938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,1024,0.06438613335291545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,10240,1024,0.06479146480560302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,10240,128,0.02308053374290466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,16384,0.7373514811197917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,16384,0.5018368085225423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,51200,1.5510602315266928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,65536,1.9911424001057942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,12288,0.5546005249023438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,51200,2.4760777791341146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,12288,0.4059893290201823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,12288,0.5349642435709636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,10240,0.33188053766886394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,10240,0.48402026494344075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,65536,3.126232655843099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,8192,0.367633056640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,8192,0.26085012753804526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,7168,0.350432014465332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,7168,0.2386101404825846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,6144,0.20362985928853355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,6144,0.2965183893839518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,5120,0.2401642640431722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,6144,0.2809877395629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,5120,0.1681546688079834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,5120,0.23957653045654298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,4096,0.1841386636098226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,4096,0.14657492637634278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,16384,0.7246837615966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,3584,0.1752768039703369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,3584,0.13525439898173014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,10240,0.46281280517578127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,3072,0.1411285400390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,3072,0.11202560265858967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,51200,2.5038922627766924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,65536,3.253275807698568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,7168,0.3376405398050944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,2560,0.13411307334899902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,2048,0.10990080038706462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,2048,0.08908159732818603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,1536,0.0777237335840861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,1536,0.06688640117645264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,1536,0.0686730702718099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,1024,0.054384001096089686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,1024,0.052553598086039224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,8192,0.38341760635375977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,768,0.044207998116811115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,768,0.04670506715774536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,768,0.04258559942245484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,512,0.03379733165105184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,512,0.03922453324000041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,4096,0.1944917360941569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,256,0.023512534300486245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,2560,0.11569066842397054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,256,0.032305065790812174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,128,0.018331732352574667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,128,0.031542400519053146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,3584,0.15865492820739746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,64,0.016290133198102318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,64,0.030331732829411824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,3072,0.13957974116007488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,8192,32,0.01702400048573812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,8192,32,0.03160213430722554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,2560,0.11867307027180989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,2048,0.09388480186462403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,1024,0.05367786486943563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,512,0.03434559901555379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,65536,1.757701365152995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,256,0.02941333254178365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,8192,128,0.02445546587308248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,16384,0.4368629455566406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,51200,2.0862164815266926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,65536,2.7822250366210937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,16384,0.6447722752888997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,51200,1.356823476155599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,12288,0.4526506741841634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,10240,0.2968384106953939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,10240,0.4102047920227051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,12288,0.372216542561849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,8192,0.3041621208190918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,8192,0.2523519992828369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,7168,0.27494827906290686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,7168,0.20918614069620767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,6144,0.23137280146280922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,6144,0.18233493169148762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,6144,0.2517226696014404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,5120,0.19413013458251954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,5120,0.15180907249450684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,16384,0.655177625020345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,4096,0.1576576073964437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,4096,0.13307092984517416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,12288,0.47579625447591145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,3584,0.1407296021779378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,65536,3.2613311767578126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,8192,0.33715947469075525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,10240,0.41419200897216796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,3584,0.11748800277709961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,3584,0.14833706219991047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,3072,0.11917653083801269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,3072,0.09760746955871583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,2560,0.10155200163523356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,2560,0.08593706289927164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,51200,2.1226080576578776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,2048,0.08497920036315917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,2560,0.10381546815236409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,7168,0.3007498741149902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,1536,0.06637226740519206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,2048,0.07325653235117593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,1536,0.060540799299875894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,1024,0.04863359928131104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,1024,0.04872746864954631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,768,0.03981333176294963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,768,0.042713598410288496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,512,0.029291733105977373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,512,0.03623253504435221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,5120,0.21153813997904458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,256,0.020550400018692017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,256,0.030136533578236896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,256,0.02681066592534383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,128,0.01577173372109731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,128,0.027702399094899494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,4096,0.17356053988138836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,64,0.014390400052070618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,3072,0.1228490670522054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,64,0.028152533372243244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,7168,32,0.01518933375676473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,1536,0.06295466820398966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,7168,32,0.028894933064778645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,2048,0.08457067012786865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,768,0.0387445330619812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,1024,0.049592534701029464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,512,0.031676799058914185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,51200,1.2324000040690104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,65536,1.6224319458007812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,7168,128,0.022358399629592896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,51200,1.8141919453938802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,65536,2.314931233723958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,16384,0.5324511845906575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,16384,0.42191572189331056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,12288,0.41771094004313153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,10240,0.3482346534729004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,12288,0.32565011978149416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,10240,0.2726880073547363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,8192,0.2612672011057536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,8192,0.21818025906880698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,7168,0.22972586949666343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,7168,0.18427093823750812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,6144,0.20179947217305502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,6144,0.15970452626546222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,6144,0.22114987373352052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,5120,0.16978880564371746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,5120,0.13557546933492023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,16384,0.5754688262939454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,4096,0.13736640612284343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,51200,1.9719445546468097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,65536,2.6322166442871096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,12288,0.4187487920125325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,3584,0.13115306695302326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,3584,0.10136853059132893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,10240,0.362771193186442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,3072,0.10442240238189697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,3072,0.09291093349456787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,8192,0.2951775868733724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,3072,0.10566826661427815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,2560,0.0902944008509318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,2048,0.07375893592834473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,2560,0.07971413135528564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,2048,0.06964373588562012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,7168,0.26632960637410485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,1536,0.057137068112691244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,1536,0.05674240191777548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,1536,0.05597013235092163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,1024,0.04092906713485718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,1024,0.04573226769765218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,768,0.03402666648228963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,768,0.04041386842727661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,4096,0.11649386882781983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,512,0.02584213415781657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,512,0.03413973251978557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,5120,0.188699738184611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,256,0.018132267395655315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,256,0.029069866736729937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,4096,0.15368213653564453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,128,0.014335999886194864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,128,0.026893866062164307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,3584,0.1264234701792399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,64,0.012505599856376648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,64,0.02688960035641988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,6144,32,0.01316159963607788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,6144,32,0.027188267310460406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,2048,0.07404053211212158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,2560,0.08993493715922038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,1024,0.043150933583577473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,768,0.034835199515024826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,512,0.028522666295369464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,65536,1.3563232421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,256,0.023822933435440063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,51200,1.079910405476888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,65536,1.8814805348714192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,51200,1.4206005096435548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,16384,0.42984854380289717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,16384,0.3475722630818685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,12288,0.29271574020385743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,12288,0.3610602696736654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,10240,0.26611305872599283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,10240,0.23598079681396483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,10240,0.31625067392985023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,8192,0.2175818602244059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,8192,0.1861130714416504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,6144,128,0.020042665799458823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,7168,0.1936245282491048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,7168,0.16222507158915203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,6144,0.16743253072102865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,6144,0.13818987210591632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,5120,0.13394559224446614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,5120,0.11540160179138184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,16384,0.48708054224650066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,12288,0.3627999941507975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,4096,0.11276799837748211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,65536,2.1026442209879557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,4096,0.09660906791687011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,51200,1.572939682006836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,3584,0.08829973538716634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,8192,0.2547082742055257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,7168,0.22315306663513185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,3072,0.08437120119730632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,3072,0.07743893464406332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,2560,0.0708085298538208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,6144,0.18632532755533854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,2560,0.06798720359802246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,2048,0.058253868420918786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,2048,0.058083200454711915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,5120,0.16017920176188152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,2048,0.05927679936091105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,1536,0.044860800107320145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,1536,0.04803733428319295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,1024,0.03208853403727214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,1024,0.038330666224161786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,3584,0.10792853037516277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,768,0.026166399319966633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,768,0.03255680004755656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,4096,0.12806613445281984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,512,0.020201599597930907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,3584,0.1103552023569743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,512,0.028700800736745198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,256,0.01418773333231608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,256,0.02430079976717631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,128,0.010433066884676616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,128,0.021614933013916017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,3072,0.094158935546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,64,0.009147733449935913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,64,0.02212266723314921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,5120,32,0.009591466188430786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,5120,32,0.02211306691169739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,2560,0.07686399618784587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,1536,0.04610986709594726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,1024,0.03822293281555176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,768,0.02874666651089986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,65536,1.20950075785319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,65536,1.519384511311849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,51200,1.0812938690185547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,16384,0.3910186767578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,51200,0.9407232284545899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,16384,0.33240105311075846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,12288,0.26059412956237793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,12288,0.24698774019877115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,512,0.023452800512313843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,256,0.018568533658981323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,10240,0.2183466593424479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,51200,1.3400608062744142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,10240,0.19886080423990887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,5120,128,0.015795200069745382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,8192,0.17603732744852701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,8192,0.1735637346903483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,7168,0.1575551986694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,7168,0.1474410692850749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,6144,0.135972261428833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,6144,0.12513279914855957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,5120,0.11215466658274334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,5120,0.1053920030593872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,5120,0.1279594659805298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,4096,0.09105172952016195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,4096,0.08863786856333414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,65536,1.7456532796223958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,3584,0.0887733300526937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,16384,0.39103358586629233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,3584,0.08886720339457194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,3072,0.0731007973353068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,3072,0.07261652946472168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,12288,0.29270505905151367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,2560,0.058481065432230625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,10240,0.2559413274129232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,2560,0.06136213143666586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,2048,0.04794666767120361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,8192,0.20205972989400228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,2048,0.05226346651713053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,1536,0.03716906706492106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,7168,0.1783690611521403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,6144,0.15221974054972331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,1536,0.0428053339322408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,1024,0.02727573315302531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,1024,0.03451626698176066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,768,0.021772799889246623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,768,0.030324266354242964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,768,0.02488853335380554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,512,0.017017600933710735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,512,0.026449066400527955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,4096,0.10289386908213298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,256,0.011779200037320454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,256,0.022809600830078124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,3584,0.08878933588663737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,128,0.00901759962240855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,128,0.021321600675582884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,3072,0.0752288023630778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,64,0.008469333251317341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,64,0.02155839999516805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,4096,32,0.009224533041318258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,2560,0.06470079819361368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,4096,32,0.0217685341835022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,2048,0.050101331869761144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,1536,0.038610132535298665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,1024,0.0327349325021108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,512,0.020093866189320884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,65536,1.095797348022461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,65536,1.283748245239258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,51200,0.8767317454020181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,256,0.016145066420237223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,65536,1.3999242146809896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,16384,0.33583361307779946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,16384,0.2951103846232096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,12288,0.2507743994394938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,12288,0.23903999328613282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,10240,0.2025749365488688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,51200,1.079144541422526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,10240,0.20292159716288247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,10240,0.21629227002461754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,8192,0.17036693890889484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,8192,0.15528640747070313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,7168,0.14969706535339355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,51200,0.9371434529622397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,4096,128,0.013697066903114319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,7168,0.1595381259918213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,6144,0.11668480237325032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,6144,0.11255040168762206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,5120,0.09831039905548096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,6144,0.12847466468811036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,5120,0.09683093229929605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,4096,0.08308693567911783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,4096,0.08092479705810547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,4096,0.09149333635965982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,3584,0.07042453289031983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,3584,0.07585813204447428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,3072,0.06063146591186523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,3072,0.06597546736399332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,3072,0.06584746837615967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,2560,0.050861867268880215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,2560,0.057394134998321536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,2560,0.05771199862162272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,2048,0.04139413436253865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,7168,0.1338666598002116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,2048,0.04720960060755412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,2048,0.04636053244272868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,1536,0.0325653334458669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,1536,0.03980586528778076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,1024,0.02395626703898112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,1024,0.03229333360989888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,1024,0.028961066404978437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,768,0.019522132476170857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,768,0.027755733331044512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,768,0.023079466819763184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,512,0.014876799782117209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,512,0.024971733490626015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,512,0.018441599607467652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,256,0.010522666573524474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,256,0.021814399957656862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,16384,0.32831573486328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,12288,0.24985920588175453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,128,0.02029119928677877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,128,0.013395200173060099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,64,0.007231999933719635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,64,0.022341332832972207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3584,32,0.0224671999613444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,8192,0.17400959332784016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,5120,0.11169173717498779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,128,0.008221866687138875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,65536,1.1516426086425782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,3584,0.07827626864115397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3584,32,0.009488000472386678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,65536,1.0266090393066407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,16384,0.26682027180989587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,16384,0.2704319953918457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,51200,0.8319530487060547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,12288,0.20120639801025392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,51200,0.8644362767537436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,1536,0.035410133997599284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,12288,0.22659626007080078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,10240,0.17201174100240071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,10240,0.17121493021647136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,8192,0.13909653027852376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,8192,0.13863466580708822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,7168,0.11817279656728108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,7168,0.12143572966257732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3584,256,0.01565439999103546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,6144,0.10283839702606201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,6144,0.10654079914093018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,12288,0.23611733118693032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,5120,0.08719253540039062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,5120,0.08980053265889486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,6144,0.12263893286387126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,4096,0.07171839872996012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,16384,0.3098826726277669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,65536,1.3530155181884767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,4096,0.07666666507720947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,3584,0.06409706672032675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,3584,0.06920426686604818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,10240,0.2050165335337321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,3072,0.05482133229573568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,3072,0.06062399943669637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,2560,0.04630933205286662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,2560,0.05206400156021118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,7168,0.14453013737996417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,2048,0.036892799536387126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,2048,0.04315840005874634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,51200,1.0445525487263998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,8192,0.16520427068074545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,1536,0.029075199365615846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,1536,0.03652373154958089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,1024,0.020544000466664634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,1024,0.0294378658135732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,3072,0.06157333453496298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,768,0.016857600212097167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,768,0.02560746669769287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,1536,0.03265173236529033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,512,0.013184000054995218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,512,0.024217599630355836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,4096,0.08486080169677734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,256,0.009291733304659527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,5120,0.10445120334625244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,256,0.020866133769353232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,128,0.0074325333038965866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,128,0.01951893369356791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,2560,0.053710933526357016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,64,0.006433066725730896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,64,0.019523199399312338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,3072,32,0.006810666620731353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,3072,32,0.019495467344919838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,3584,0.07449813683827719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,2048,0.04094613393147786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,65536,0.8553034464518229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,65536,0.916041628519694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,51200,0.6969216028849284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,65536,1.037282117207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,51200,0.7110026677449544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,16384,0.21031573613484703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,16384,0.24018026987711588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,12288,0.15819519360860188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,12288,0.18368746439615885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,12288,0.196452267964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,10240,0.13216959635416667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,1024,0.0256330668926239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,10240,0.15518186887105306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,8192,0.10654186407725016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,8192,0.12520960172017415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,128,0.012386133273442585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,7168,0.09299840132395426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,7168,0.11045440038045247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,256,0.014500266313552857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,6144,0.08010240395863852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,6144,0.09634346961975097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,16384,0.25354666709899903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,512,0.017113600174585977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,5120,0.06844586531321208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,5120,0.08418560028076172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,4096,0.056251732508341465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,3072,768,0.020553600788116456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,4096,0.06956799825032553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,4096,0.0692629337310791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,3584,0.04973119894663493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,3584,0.062021334966023765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,3072,0.042209064960479735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,3072,0.05415680011113485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,3072,0.05043306748072306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,2560,0.035923198858896895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,2560,0.04642133315404256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,10240,0.16548906962076823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,2048,0.029578665892283123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,2048,0.039214932918548585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,2048,0.035267198085784913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,51200,0.7971754709879557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,1536,0.02365866700808207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,8192,0.13255573113759359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,1536,0.02712213397026062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,1536,0.034213332335154216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,1024,0.017820799350738527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,768,0.014961066842079162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,1024,0.027857067187627156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,1024,0.02290560007095337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,512,0.011714133620262145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,768,0.02504533330599467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,512,0.02231360077857971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,256,0.008605866630872091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,256,0.02017813324928284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,256,0.011935999989509583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,128,0.0071050668756167095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,128,0.018910932540893554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,7168,0.11683200200398762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,64,0.006229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,64,0.018578133980433145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2560,32,0.006587733328342438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,6144,0.09792106946309408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2560,32,0.01867413322130839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,5120,0.08338027000427246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,65536,0.6746623992919922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,3584,0.05968533356984457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,65536,0.8357631683349609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,51200,0.5855690638224285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,65536,0.9624330520629882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,51200,0.6626410802205404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,16384,0.19096213976542156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,2560,0.04395093520482381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,12288,0.1454901377360026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,16384,0.23887786865234376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,10240,0.11020159721374512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,12288,0.16754026412963868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,768,0.01726079980532328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,512,0.01399786671002706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,10240,0.14137813250223796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,8192,0.08888320128122965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,8192,0.11519467035929362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,7168,0.07738347053527832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2560,128,0.010193066795667012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,7168,0.10221227010091145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,6144,0.06685973008473714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,6144,0.08898133436838786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,16384,0.24171627362569176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,51200,0.750164286295573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,5120,0.05786240100860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,12288,0.18230506579081218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,4096,0.04582080046335856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,5120,0.07770773569742838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,5120,0.0797930637995402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,10240,0.1565824031829834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,3584,0.04150720040003459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,4096,0.06309546629587809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,4096,0.06360640128453574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,3584,0.05904533465703329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,3072,0.036194133758544925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,2560,0.03189226587613424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,3072,0.04870186646779378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,2560,0.04270399808883667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,2048,0.0249674657980601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,2048,0.03683520158131917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,8192,0.12051733334859212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,1536,0.020489599307378134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,1536,0.030703999598821002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,1024,0.015053866306940713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,7168,0.10586026509602864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,6144,0.0923520008722941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,768,0.01245013376077016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,768,0.023014400402704874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,768,0.016407466928164163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,512,0.00976213316122691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,512,0.020853332678476968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,1024,0.026873600482940675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,256,0.0073525334397951765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,256,0.019393066565195717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,3584,0.054017066955566406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,3072,0.04676053524017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,128,0.006212266782919565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,128,0.010148266951243084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,128,0.01826559901237488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,64,0.005709866682688395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,2048,32,0.006431999802589417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,64,0.01856000026067098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,2048,0.032781867186228435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,1536,0.02550613284111023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,2048,32,0.018551466862360637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,1024,0.020499199628829956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,51200,0.39974826176961265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,65536,0.5159530639648438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,16384,0.1313546657562256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,65536,0.7569760004679362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,16384,0.1984608014424642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,51200,0.5952565511067708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,16384,0.18558932940165201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,12288,0.10274666945139568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,12288,0.15279679298400878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,10240,0.08321812947591146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,12288,0.13964160283406574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,512,0.013766400019327798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,8192,0.06812266508738199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,10240,0.13021546999613445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,2560,0.040852268536885575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,7168,0.06004800001780192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,8192,0.10528960227966308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,6144,0.05135253270467123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,7168,0.09340373675028482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,6144,0.08114240169525147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,6144,0.07449386914571127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,5120,0.043346134821573894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,5120,0.06833279927571614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,5120,0.06228053172429403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,4096,0.03561066786448161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,2048,256,0.011623467008272808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,4096,0.055567999680837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,3584,0.03161919911702474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,3584,0.04305599927902222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,3584,0.048818135261535646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,3072,0.02812906702359517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,3072,0.042624000708262125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,2560,0.023653332392374674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,2560,0.037264001369476316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,65536,0.7383136113484701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,2048,0.020755199591318767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,2048,0.03465493520100911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,51200,0.577125358581543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,1536,0.01602453291416168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,1536,0.027769599358240766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,10240,0.12548480033874512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,1536,0.02090880076090495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,1024,0.01197653313477834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,1024,0.023558400074640908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,8192,0.09683413505554199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,7168,0.08503572940826416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,768,0.021821866432825722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,4096,0.05141866604487101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,512,0.008150400221347808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,512,0.02033066749572754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,3072,0.03677866856257121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,256,0.006253866851329804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,256,0.018127999703089395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,2560,0.03365226586659749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,128,0.0052704001466433205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,128,0.01698346734046936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,768,0.01014293332894643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,128,0.008539733290672303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,64,0.004766933123270671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1536,32,0.004946133494377137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,64,0.016976000865300496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1536,32,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,65536,0.3728480021158854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,65536,0.6918282826741536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,51200,0.2925930658976237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,2048,0.026181334257125856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,1024,0.01625386675198873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,51200,0.5533642450968425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,16384,0.12306986649831135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,12288,0.08866346677144368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,16384,0.18289066950480143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,768,0.01421333352724711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,10240,0.08007466793060303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,12288,0.13941866556803387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,10240,0.1189802646636963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,8192,0.05464640061060587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,512,0.011660800377527872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,8192,0.09543253580729166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,7168,0.04851626555124919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1536,256,0.009841066598892213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,7168,0.08411413033803304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,6144,0.04267839988072713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,6144,0.07208746274312337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,12288,0.13146133422851564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,5120,0.035444267590840656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,5120,0.06008640130360922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,65536,0.6907306671142578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,4096,0.02523306608200073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,4096,0.047490131855010984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,16384,0.178002134958903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,3584,0.023718400796254476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,3584,0.04237653414408366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,6144,0.06763199965159097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,3072,0.019792000452677407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,51200,0.53111146291097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,10240,0.11224746704101562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,3072,0.03776853481928508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,2560,0.017196800311406454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,2560,0.03328106602032979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,2048,0.014668800433476768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,2048,0.02905600070953369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,7168,0.07742933432261148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,1536,0.011560533444086711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,1536,0.02544533411661784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,8192,0.09061973094940186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,1024,0.008899199962615966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,1024,0.021919999519983926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,3072,0.03365866740544637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,768,0.007703466713428498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,768,0.020151466131210327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,1536,0.019035732746124266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,512,0.0064533332983652755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,512,0.01907306710879008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,4096,0.04848213195800781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,256,0.005341866612434387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,5120,0.056435199578603115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,256,0.018017067511876424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,128,0.004706133405367533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,128,0.01669013301531474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,3584,0.03929493427276611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,64,0.004252799848715464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,64,0.016354133685429893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,1024,32,0.0046079998215039575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,2048,0.023306665817896526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,1024,32,0.01641493340333303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,65536,0.2760298728942871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,1024,0.014731733004252115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,65536,0.6359104156494141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,2560,0.029682133595148725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,51200,0.2150335947672526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,16384,0.07065599759419759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,51200,0.5028682708740234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,16384,0.16741226514180502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,16384,0.13344106674194336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,12288,0.053660798072814944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,12288,0.12942506472269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,10240,0.044999468326568606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,10240,0.11830080350240071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,65536,0.49289385477701825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,8192,0.03694933255513509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,8192,0.0954645315806071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,51200,0.3839968045552572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,8192,0.06785386403401693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,7168,0.032696533203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,7168,0.08435200055440267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,256,0.008804266651471455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,12288,0.09884693622589111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,6144,0.027898667256037395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,128,0.007694933315118153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,6144,0.071833602587382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,512,0.010260267059008281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,5120,0.023341866334279378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,5120,0.04557226498921712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,5120,0.0588703989982605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,4096,0.04674453337987264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,10240,0.08359893163045248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,3584,0.02228053410847982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,3584,0.03284586668014526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,3584,0.041076266765594484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,3072,0.019463467597961425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,1024,768,0.012482133507728577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,3072,0.03613866567611694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,2560,0.01683626572291056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,2560,0.031933865944544476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,2048,0.014105600118637086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,2048,0.029167999823888142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,2048,0.020406399170557657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,1536,0.011122133334477742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,1536,0.02471253275871277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,4096,0.023884799083073935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,1024,0.008585600058237712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,1024,0.022204800446828207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,7168,0.06029119888941446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,768,0.007053866485754649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,768,0.01919040083885193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,6144,0.05160746574401855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,512,0.005793066819508871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,512,0.018407466014226277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,512,0.008633599678675333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,256,0.004624000191688538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,4096,0.03708159923553467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,256,0.017259732882181803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,128,0.004002133260170618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,128,0.016264533003171287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,64,0.003688533355792364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,3072,0.0281056006749471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,64,0.016520532965660095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,768,32,0.003931733220815659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,768,32,0.01591679950555166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,65536,0.19199466705322266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,1536,0.016226133704185484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,51200,0.1489674727121989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,65536,0.5958869298299153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,65536,0.463973331451416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,16384,0.07517653306325277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,51200,0.4672106742858887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,51200,0.35717973709106443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,16384,0.1655786673227946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,256,0.007392000158627827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,12288,0.057042133808135984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,16384,0.12247040271759033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,10240,0.049510399500528976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,12288,0.13002666632334392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,1024,0.012481066584587096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,8192,0.03367146650950114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,10240,0.07806933720906575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,10240,0.11238933404286702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,2560,0.024421334266662598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,7168,0.03312746683756511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,8192,0.08672320048014323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,7168,0.07759573459625244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,6144,0.029193600018819172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,5120,0.025663999716440837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,6144,0.04824533462524414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,6144,0.0647157351175944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,4096,0.018463999032974243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,5120,0.052342398961385095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,4096,0.04153813521067302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,3584,0.017066667477289833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,3584,0.036429866154988604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,768,0.010368000467618306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,3072,0.014282666643460593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,3072,0.033187200625737504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,3072,0.02574613293011983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,2560,0.012055466572443645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,2560,0.029000532627105714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,768,128,0.006366933385531108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,2048,0.010414933164914448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,2048,0.02623680035273234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,12288,0.09179519812266032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,1536,0.008730666836102803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,8192,0.06462080081303914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,1536,0.024076799551645912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,1024,0.006266666452089946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,1536,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,7168,0.05533973375956217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,1024,0.020218666394551596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,768,0.0056309332450230915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,768,0.01861226757367452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,5120,0.04129173358281453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,512,0.0047765334447224935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,512,0.018318933248519898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,256,0.00408746674656868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,4096,0.034186665217081705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,256,0.016773333152135216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,3584,0.029976532856623335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,128,0.003688533355792364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,128,0.016083199779192606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,2560,0.02249493400255839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,64,0.0034261333445707956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,64,0.015682133038838704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,512,32,0.003651199986537298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,768,0.009711999694506328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,512,32,0.015531733632087708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,65536,0.12050346533457439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,2048,0.018942934274673463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,51200,0.09566826820373535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,65536,0.5603125254313152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,51200,0.44051307042439775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,16384,0.0371018648147583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,16384,0.1548917293548584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,1024,0.011716266473134358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,12288,0.03529920180638631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,512,0.00816319982210795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,12288,0.12011733055114746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,256,0.007010133564472198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,10240,0.030035199721654256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,8192,0.023924267292022704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,10240,0.10324266751607258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,8192,0.08272960186004638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,7168,0.0219925324122111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,7168,0.07157333691914877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,512,128,0.00588266650835673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,6144,0.019246933857599895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,6144,0.06000746488571167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,65536,0.41416533788045246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,5120,0.015179733435312906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,5120,0.04762453238169352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,5120,0.03751680056254069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,51200,0.32539841334025066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,4096,0.01242453356583913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,4096,0.03789226611455281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,3584,0.011778133114178975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,3584,0.034219733874003094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,16384,0.11027946472167968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,3072,0.009896533687909444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,3072,0.03212053378423055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,12288,0.08340160051981607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,2560,0.008880000313123066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,10240,0.07105173269907633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,2560,0.027498666445414228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,8192,0.057811200618743896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,2048,0.007520000139872233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,2048,0.02454400062561035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,1536,0.006444799900054932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,1536,0.02217280069986979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,7168,0.050911998748779295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,1024,0.005169066786766052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,1024,0.019497599204381308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,6144,0.04452053308486938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,768,0.00462719996770223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,768,0.017988266547520955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,4096,0.02990079919497172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,512,0.004103466620047887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,512,0.01813973387082418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,512,0.007672533392906189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,256,0.003642666588226954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,3584,0.02659733295440674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,256,0.016173866391181946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,128,0.0033930666744709016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,3072,0.023404800891876222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,128,0.01541866660118103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,64,0.0031648000081380212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,64,0.01532266636689504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,256,32,0.003190399954716364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,256,32,0.015333333611488342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,65536,0.10789546966552735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,2560,0.020574933290481566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,65536,0.5493920008341472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,2048,0.01717546582221985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,51200,0.08611520131429037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,1536,0.014100266496340432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,16384,0.031540266672770184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,1024,0.010756267110506694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,51200,0.2917429288228353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,16384,0.1500757376352946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,16384,0.09845866362253824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,51200,0.4354698816935222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,12288,0.025483733415603636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,10240,0.022204800446828207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,768,0.009051733215649923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,12288,0.0750922679901123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,12288,0.11556479930877686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,10240,0.06319679816563925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,8192,0.01577600042025248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,7168,0.01423679987589518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,8192,0.07864747047424317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,6144,0.0123989333709081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,7168,0.06760213375091553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,7168,0.0463701327641805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,5120,0.010941867033640544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,6144,0.04100159804026286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,5120,0.04494080146153768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,4096,0.009307733178138733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,4096,0.03572479883829753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,256,0.006537599861621857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,4096,0.027370667457580565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,256,128,0.005629866818586985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,10240,0.09758079846700032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,3584,0.008629332979520161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,8192,0.052293332417805996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,3584,0.024401066700617473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,3584,0.031541333595911665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,3072,0.029575467109680176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,2560,0.02658453385035197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,2560,0.018589866161346436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,2048,0.006380799909432728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,2048,0.024421334266662598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,65536,0.37666985193888347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,1536,0.005596800148487091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,1536,0.021769599119822184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,1536,0.012852266430854797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,3072,0.0079925333460172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,1024,0.004636799792448679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,1024,0.019319466749827065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,2560,0.007201066613197327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,768,0.004286933441956838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,768,0.017901867628097534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,6144,0.05626453161239624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,512,0.003824000060558319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,512,0.01734293301900228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,256,0.003340800106525421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,256,0.016101333498954772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,256,0.0061471998691558834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,128,0.0030901332696278887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,128,0.01552959978580475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,128,0.005277866621812185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,5120,0.03444266716639201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,64,0.0030239999294281008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,128,32,0.003078400095303853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,64,0.015058133006095886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,128,32,0.014912000298500061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,65536,0.10305600166320801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,51200,0.07942933241526286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,16384,0.030727465947469074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,65536,0.5452351888020833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,51200,0.4299317359924316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,12288,0.024078933397928874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,16384,0.14840426445007324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,10240,0.01958720088005066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,8192,0.013934933145840964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,12288,0.11371413071950276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,3072,0.021447465817133585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,10240,0.09589227040608725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,7168,0.012880000472068786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,8192,0.07794666290283203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,6144,0.01028053363164266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,5120,0.009955199559529622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,7168,0.06720320383707681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,4096,0.008543999989827473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,6144,0.05479893287022909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,5120,0.043697067101796466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,4096,0.03466666539510091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,3584,0.032586665948232015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,2560,0.006506666541099548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,3072,0.028936533133188884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,2560,0.025991467634836833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,2048,0.01597119967142741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,2048,0.024306132396062215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,1536,0.005022933085759481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,1024,0.004257066547870636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,1536,0.02145386735598246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,768,0.003920000046491623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,1024,0.019117865959803262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,512,0.0035562666753927866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,768,0.017810134092966716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,256,0.0032277333239714304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,512,0.01717546582221985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,128,0.002921599894762039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,256,0.015733333428700765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,64,0.0028437333802382152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,128,0.015355733036994935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,32,0.0028384000062942503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,64,0.014872533082962037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,64,32,0.015007999539375306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,65536,0.10828266938527424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,51200,0.08305493195851645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,1024,0.010089600086212158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,16384,0.03060693343480428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,768,0.008370133241017659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2049,128,512,0.007192533214886982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,16384,0.14641280174255372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,65536,0.5483306884765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,10240,0.016406400005022685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,3072,0.007157333195209503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,12288,0.11280639966328938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,8192,0.011040000120798747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,51200,0.43015146255493164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,7168,0.009798399607340495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,8192,0.07641920248667398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,6144,0.008784000078837078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,2048,0.0056415999929110205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,7168,0.06571306784947714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,5120,0.009637332955996196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,4096,0.008389332890510559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,3584,0.007569066683451335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,5120,0.04384640057881673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,4096,0.03549226522445679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,3072,0.00702400008837382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,3584,0.031640533606211343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,3072,0.029895466566085816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,2560,0.006592000027497609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,2560,0.026286933819452924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,2048,0.0058794667323430385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,2048,0.02408000032107035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,1536,0.0050784001747767125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,1024,0.004257066547870636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,1536,0.021991467475891112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,768,0.004007466634114583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,1024,0.019614932934443156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,768,0.018041600783665977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,512,0.0036277333895365395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,256,0.003389866650104523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,512,0.017083734273910522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,128,0.003180799881617228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,256,0.016028799613316855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,64,0.0030464000999927522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,128,0.015414399902025857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,32,0.0030165334542592366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,64,0.01479680041472117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,32,0.015220266580581666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,32,12288,0.02325973312060038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2049,64,3584,0.00795306662718455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,10240,0.09597973028818765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2049,32,6144,0.05386666854222616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,16384,3.2916404724121096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,16384,3.8982688903808596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,12288,2.621068827311198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,12288,4.61347401936849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,16384,6.011069742838542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,12288,3.0212865193684895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,10240,2.13165651957194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,10240,2.41209716796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,10240,3.691597747802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,8192,1.575332260131836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,8192,3.0247166951497393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,8192,2.0835904439290363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,7168,1.4109376271565757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,51200,9.808519490559895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,7168,2.754590861002604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,7168,1.7810634613037108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,6144,1.27629763285319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,5120,0.9615264256795248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,6144,2.267969004313151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,6144,1.5324320475260416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,5120,1.2642954508463542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,4096,0.8129589080810546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,5120,2.2200948079427083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,4096,1.5454485575358072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,4096,1.1194688161214192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,3584,0.820691172281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,3584,1.416696548461914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,3584,0.9141941070556641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,3072,0.6454399744669597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,3072,0.9215850830078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,2560,0.6128405253092448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,3072,1.2897120157877604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,2560,1.1347860972086588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,51200,15.108802286783853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,2560,0.7558783849080404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,2048,0.5012650807698568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,1536,0.35645227432250975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,2048,0.6926037470499675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,1536,0.6277621587117512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,1024,0.49223893483479814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,1024,0.28073921203613283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,1024,0.36723947525024414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,768,0.24154987335205078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,768,0.3607103983561198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,1536,0.49087146123250325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,512,0.23109119733174643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,512,0.17279040018717448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,256,0.16920426686604817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,768,0.40636587142944336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,512,0.28577280044555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,128,0.11751253604888916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,256,0.1409781297047933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,128,0.12330880165100097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,256,0.22899840672810873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,64,0.13018346627553304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,32,0.11340373357137043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,64,0.12051946322123211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,128,0.2622624079386393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,32,0.11930346488952637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,2048,0.7821919759114583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,51200,20.205438232421876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,16384,4.863505045572916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,16384,2.645088958740234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,16384,3.26685536702474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,51200,8.019584147135417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,12288,3.7547327677408853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,12288,1.9891882578531903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,51200,10.456552124023437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,65536,10.245679728190105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,12288,2.2900065104166667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,10240,1.7014432271321613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,10240,3.307760111490885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,10240,2.325067647298177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,8192,2.785559590657552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,8192,1.5581087748209634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,8192,1.6008416493733724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,7168,1.1824095408121744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,65536,16.31064656575521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,51200,16.152945963541665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,7168,1.4705248514811198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,6144,0.9699125289916992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,6144,1.8697728474934894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,6144,1.1136906941731772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,5120,0.8119050979614257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,7168,2.213724772135417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,4096,0.6903829574584961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,5120,1.445526377360026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,4096,0.814574940999349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,4096,1.2479957580566405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,3584,0.5590037027994792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,3584,1.0657621383666993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,3584,0.7065642674763997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,3072,0.5253408114115398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,3072,0.9572895685831705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,3072,0.6252960205078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,2560,0.4510815938313802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,2560,0.5174421310424805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,2560,0.7827615737915039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,2048,0.6175637563069661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,2048,0.38107306162516275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,2048,0.4475861231486003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,1536,0.2761397361755371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,1024,0.221891196568807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,1536,0.4689503987630208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,1024,0.35118932723999025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,1536,0.4035296122233073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,1024,0.2831946690877279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,768,0.2432096004486084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,768,0.17919146219889323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,512,0.19773866335550944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,256,0.12090026537577311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,512,0.14528533617655437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,512,0.20397653579711914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,256,0.1090997298558553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,128,0.09418133099873861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,128,0.10113920370737713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,64,0.08711360295613607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,256,0.18656427065531414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,64,0.14499200185139974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,32,0.13263040383656818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,51200,32,0.14497920672098796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,51200,65536,20.48563028971354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,5120,0.9366122563680014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,128,0.15282880465189616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,16384,1.521019744873047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,51200,2.8307851155598955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,65536,3.5432406107584633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,16384,0.8787935892740885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,51200,3.202955627441406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,65536,4.012795766194662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,16384,0.9744202931722006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,12288,0.6510325113932292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,51200,4.654265848795573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,12288,1.0683648427327475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,12288,0.7587786356608073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,10240,0.58482240041097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,10240,0.9424298604329427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,10240,0.6003157297770183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,8192,0.4473589261372884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,8192,0.5173194567362468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,8192,0.711842155456543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,7168,0.4010101318359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,7168,0.46846507390340164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,7168,0.6615594863891602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,65536,5.9151453653971355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,6144,0.36939306259155275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,6144,0.5490698496500651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,6144,0.36088107426961263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,5120,0.3056703885396322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,5120,0.4581311861673991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,5120,0.33421548207600915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,4096,0.25100372632344564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,3584,0.20886507034301757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,4096,0.4036992073059082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,3584,0.3107093175252279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,3584,0.24738027254740397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,2560,0.2504480044047038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,3072,0.2717162768046061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,2560,0.16805226008097332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,3072,0.182205867767334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,3072,0.19569387435913085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,2048,0.20397866566975914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,1536,0.14240533510843914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,2048,0.13624746004740398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,2048,0.1459658622741699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,1024,0.10014399687449138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,1024,0.0825984001159668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,1024,0.09540159702301025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,768,0.07941866715749105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,1536,0.1363807996114095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,512,0.058525868256886805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,768,0.07045119603474935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,512,0.055829334259033206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,512,0.06954987049102783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,256,0.037962667147318524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,256,0.04851306676864624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,256,0.057309865951538086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,51200,768,0.2275829315185547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,128,0.03303999900817871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,64,0.030535467465718585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,128,0.04389653205871582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,128,0.052279468377431235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,32,0.030929066737492877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,64,0.04544426600138347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,32,0.045797332127888994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,2560,0.18254292805989583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,1536,0.10461440086364746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,768,0.08044479688008627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,51200,2.08483517964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,65536,2.728830973307292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,65536,3.094976043701172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,51200,2.3439956665039063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,16384,0.7015189488728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,16384,1.0962570190429688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,16384,0.7219615936279297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,51200,3.5495051066080725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,12288,0.5083381334940593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,12288,0.5433184305826824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,12288,0.849127451578776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,10240,0.47356586456298827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,10240,0.7098239898681641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,10240,0.4622559865315755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,65536,4.47400868733724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,8192,0.5415679931640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,8192,0.37047786712646485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,7168,0.31197760899861654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,7168,0.48694826761881516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,7168,0.330949338277181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,6144,0.28219839731852214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,6144,0.28480958938598633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,6144,0.43017174402872727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,5120,0.2368021329243978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,5120,0.32782293955485026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,4096,0.28341973622639977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,4096,0.20869973500569663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,3584,0.17529279390970867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,3072,0.21612906455993652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,3072,0.15432747205098468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,3072,0.1684351921081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,2560,0.1692021369934082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,2560,0.13326186339060467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,2560,0.13420480092366535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,2048,0.14430079460144044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,2048,0.10875840187072754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,2048,0.11678826808929443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,1536,0.10943360328674316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,1536,0.08686827023824056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,1024,0.07636160055796305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,5120,0.2502687931060791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,4096,0.19739093780517578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,1024,0.06628053188323975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,768,0.059229866663614905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,1024,0.07050773302714029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,512,0.04506133397420247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,768,0.06266560157140097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,3584,0.23514560063680015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,512,0.047413333257039385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,512,0.052527999877929686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,256,0.040601599216461184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,256,0.044172799587249754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,128,0.025808000564575197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,128,0.037912531693776445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,4096,0.24293333689371743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,128,0.04143146673838298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,64,0.02405973275502523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,32,0.02536639968554179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,64,0.03841813405354817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,32,0.038943998018900555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,1536,0.0934613307317098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,768,0.05753173430760702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,256,0.030444800853729248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,65536,2.3659828186035154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,65536,2.5528704325358076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,51200,1.8725643157958984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,51200,1.9643850962320966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,16384,0.633140246073405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,16384,0.9178431828816732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,51200,2.960875701904297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,65536,3.808674112955729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,16384,0.6522176106770833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,12288,0.7209749221801758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,12288,0.4548298517862956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,10240,0.5795487721761068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,10240,0.38706881205240884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,10240,0.41857493718465166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,8192,0.3129023869832357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,8192,0.47695465087890626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,8192,0.3191381454467773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,7168,0.2713322639465332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,7168,0.29925012588500977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,7168,0.3847071965535482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,6144,0.3464405377705892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,6144,0.23383785883585612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,6144,0.2338816006978353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,5120,0.2897493362426758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,5120,0.20223679542541503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,5120,0.19847359657287597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,4096,0.2272576014200846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,4096,0.1713696002960205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,4096,0.1712480068206787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,3584,0.19515306154886883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,12288,0.46712961196899416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,3584,0.15312320391337078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,3584,0.14747519493103028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,3072,0.13012693723042806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,3072,0.1719231923421224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,3072,0.12435413201649984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,2560,0.11261546611785889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,2048,0.12097386519114177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,2048,0.09264427026112874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,2560,0.11221226851145427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,2048,0.09952426751454671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,1536,0.09431573549906412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,1536,0.07577386697133383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,1024,0.0667253335316976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,1024,0.06425600051879883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,1024,0.06078506708145141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,768,0.05361173152923584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,768,0.05562986532847086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,512,0.041094398498535155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,768,0.05391039848327637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,256,0.028274132808049517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,512,0.04600106477737427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,512,0.04597760041554769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,128,0.0225983997186025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,256,0.03641066551208496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,128,0.03400853474934896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,128,0.03563626607259114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,64,0.020111999909083047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,32,0.020376533269882202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,64,0.03312639991442363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,32,0.0342901349067688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,2560,0.14671999613444012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,8192,0.35609814325968425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,1536,0.07966826756795248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,3584,0.16623679796854657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,256,0.038747731844584146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,16384,0.7055957158406575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,16384,0.4889845212300618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,65536,1.9441824595133466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,51200,1.4774548848470053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,16384,0.5055935859680176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,12288,0.5269002596537272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,51200,1.5634656270345053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,12288,0.3850943883260091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,65536,2.1258389790852865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,65536,2.9931177775065105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,12288,0.3766773223876953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,10240,0.3304778734842936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,8192,0.35699733098347985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,51200,2.3987050374348957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,10240,0.4755071957906087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,10240,0.34597225189208985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,8192,0.27915306091308595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,7168,0.2366741339365641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,7168,0.32036266326904295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,7168,0.232808526357015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,6144,0.21290879249572753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,6144,0.21814932823181152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,5120,0.23317440350850424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,6144,0.29733867645263673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,5120,0.1750922679901123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,5120,0.16842133204142254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,4096,0.1891317367553711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,4096,0.14522347450256348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,3584,0.16172693570454916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,3584,0.12877439657847087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,3584,0.1234282652537028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,3072,0.10714879830678303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,3072,0.11597653230031331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,2560,0.09433813095092773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,2560,0.0948352018992106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,2048,0.09525333245595297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,2048,0.08463786443074545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,1536,0.0727018674214681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,1024,0.052551468213399254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,1536,0.07031573454538981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,1024,0.05236053466796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,1024,0.05564586718877157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,768,0.04166719913482666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,768,0.04848853349685669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,768,0.04454506635665893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,512,0.03458666801452637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,512,0.03926080067952474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,512,0.039374931653340654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,256,0.022275199492772423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,256,0.033666133880615234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,256,0.031185066699981688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,128,0.018655999501546224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,128,0.02932159900665283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,128,0.02993920048077901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,64,0.014827733238538107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,8192,0.2710965474446615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,64,0.03038933277130127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,32,0.01560640037059784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,32,0.030011733373006184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,4096,0.1449834664662679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,2560,0.12071893215179444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,3072,0.13942400614420575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,1536,0.07994346618652344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,51200,1.3393739064534507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,65536,1.7437503814697266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,65536,1.8483530680338542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,51200,2.117969004313151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,16384,0.6112576166788737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,16384,0.44893547693888347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,65536,2.6445632934570313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,16384,0.504526933034261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,12288,0.32967999776204426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,12288,0.37105814615885413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,10240,0.40677865346272785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,10240,0.3011093457539876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,10240,0.2886591911315918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,8192,0.3176927884419759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,8192,0.23077653249104818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,8192,0.24170986811319986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,7168,0.2696714719136556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,7168,0.2139359951019287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,7168,0.20288853645324706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,6144,0.22482453982035316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,6144,0.18427519798278807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,5120,0.1509493350982666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,5120,0.14210559527079264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,4096,0.15626880327860515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,4096,0.12441813151041667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,4096,0.11826879978179931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,3584,0.13747307459513347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,3584,0.10807039737701415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,3584,0.10634346803029378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,3072,0.12090240319569905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,3072,0.09488000075022379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,12288,0.4591434796651204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,2560,0.10086507002512615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,3072,0.09380373160044352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,2560,0.08119466304779052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,2048,0.0819754679997762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,2048,0.06822826862335205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,2048,0.07068586349487305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,1536,0.06383786598841348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,1536,0.05599573453267416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,1024,0.04431573152542114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,1536,0.05840320189793905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,1024,0.043671464920043944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,768,0.03344426552454631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,1024,0.05170773267745972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,512,0.025693867603937787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,768,0.04123200178146362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,768,0.038466131687164305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,256,0.017910399039586387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,512,0.03423466682434082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,256,0.029028266668319702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,128,0.014544000228246054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,5120,0.19337387084960939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,128,0.02736426591873169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,64,0.013895466923713684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,128,0.026784000794092815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,32,0.014731733004252115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,64,0.0281056006749471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,32,0.02812906702359517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,512,0.0340010682741801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,256,0.030323199431101483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,65536,1.552130126953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,2048,0.07721066474914551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,65536,2.301016489664714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,65536,1.5848159790039062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,51200,1.7689183553059895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,16384,0.5661834716796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,51200,1.2287456512451171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,16384,0.46113065083821614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,12288,0.42397546768188477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,16384,0.3880864143371582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,51200,1.2139466603597007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,12288,0.3183605194091797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,10240,0.3407562573750814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,10240,0.27529172897338866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,10240,0.27483946482340493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,8192,0.26018452644348145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,51200,1.3569461822509765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,8192,0.20977279345194497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,7168,0.249016539255778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,7168,0.1862720012664795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,6144,0.198308261235555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,7168,0.19624640146891276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,6144,0.14882453282674152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,6144,0.17477866808573406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,5120,0.17096853256225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,5120,0.15389973322550457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,4096,0.1376853307088216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,4096,0.11524906953175862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,4096,0.11869973341623943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,3584,0.10645120143890381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,3584,0.09416000048319498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,3072,0.10269866784413655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,3072,0.09198400179545084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,8192,0.2234272003173828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,3072,0.08151573340098063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,2560,0.08744640350341797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,2560,0.07857813040415446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,2048,0.0711296002070109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,2560,0.07014506657918294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,2048,0.06053760051727295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,2048,0.06721386909484864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,1536,0.0555242657661438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,1536,0.056150400638580324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,1536,0.05010026693344116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,1024,0.039691734313964847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,3584,0.12184106508890788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,1024,0.0396725336710612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,768,0.032840534051259355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,768,0.03909333149592082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,768,0.03528853257497151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,512,0.02518080075581868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,512,0.0337066650390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,2560,0.07976746559143066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,256,0.018038400014241538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,256,0.028511999050776164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,256,0.02482346693674723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,128,0.01402239998181661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,128,0.02664639949798584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,128,0.022589866320292154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,64,0.01209386686484019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,32,0.01259519954522451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,64,0.026437334219614667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,32,0.02577706575393677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,1024,0.04482560157775879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,6144,0.16839680671691895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,65536,1.4050336201985678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,65536,1.9033247629801433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,65536,1.3253684997558595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,16384,0.48257919947306316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,51200,1.4293685913085938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,16384,0.37445014317830405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,51200,1.1095466613769531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,16384,0.34619414011637367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,12288,0.3664725303649902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,12288,0.2814453442891439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,10240,0.2885600090026855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,12288,0.26778240203857423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,10240,0.24955306053161622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,10240,0.20969173113505044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,8192,0.19597439765930175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,8192,0.17713066736857097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,7168,0.19029226303100585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,7168,0.15491092999776204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,6144,0.1680437405904134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,6144,0.14275306065877277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,6144,0.13682880401611328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,5120,0.1360714594523112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,5120,0.12364373207092286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,5120,0.11131839752197266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,4096,0.11457066535949707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,4096,0.09793919722239176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,4096,0.09351253509521484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,3584,0.0974176009496053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,3584,0.09118506908416749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,3584,0.08172266483306885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,3072,0.08568533261617026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,3072,0.07853439648946127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,3072,0.07216959794362386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,2560,0.07143466472625733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,8192,0.23821120262145995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,12288,0.2774314562479655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,2560,0.06962666511535645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,2560,0.061736532052358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,2048,0.05902933279673258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,2048,0.05916266838709513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,1536,0.04576853513717651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,2048,0.05368533531824747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,7168,0.17056105931599935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,1024,0.03319466710090637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,1536,0.04450240135192871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,1024,0.0383349339167277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,768,0.026808534065882367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,768,0.03320320049921672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,512,0.02040426731109619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,768,0.03118720054626465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,5120,0.12556479771931967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,256,0.014366933703422546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,512,0.029360000292460126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,128,0.010743467013041179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,256,0.021944532791773476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,256,0.024654932816823325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,64,0.009227733810742695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,128,0.019814399878184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,128,0.02261120080947876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,32,0.009450667103131612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,64,0.022418133417765298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,32,0.021997867027918498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,512,0.02964800000190735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,1536,0.04914453426996867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,1024,0.035631998380025225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,65536,1.090557861328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,512,0.026038400332132977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,65536,1.1782272338867188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,65536,1.4888309478759765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,16384,0.33572800954182946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,16384,0.3148991902669271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,51200,1.0605951944986978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,12288,0.2581066608428955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,16384,0.3078410784403483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,12288,0.24100906054178872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,51200,0.9376938501993815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,12288,0.239355738957723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,10240,0.23155199686686195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,10240,0.17306666374206542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,10240,0.21049920717875162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,8192,0.17552746136983235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,8192,0.15608426729838054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,8192,0.17236587206522624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,7168,0.14539413452148436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,7168,0.12917439937591552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,6144,0.10877226988474528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,6144,0.1253333330154419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,5120,0.11189333597819011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,5120,0.09406186739603678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,5120,0.10435946782430013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,4096,0.09070400396982828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,3584,0.08006186485290527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,4096,0.08794453144073486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,3584,0.06800959904988607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,3584,0.08126400311787924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,3072,0.06763733228047689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,3072,0.07234453360239665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,3072,0.05934079885482788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,2560,0.0608789324760437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,2560,0.05181440114974976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,2048,0.04689813454945882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,51200,0.8973013559977213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,2048,0.050885331630706784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,2048,0.04497173229853312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,1536,0.036505599816640213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,1536,0.04239039818445842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,7168,0.1514698664347331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,1536,0.037003731727600096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,1024,0.026343466838200887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,1024,0.03404373327891032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,1024,0.02945706645647685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,6144,0.1427445411682129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,768,0.029657600323359172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,512,0.016588800152142844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,512,0.02606826623280843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,512,0.02140053311983744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,256,0.011590400338172912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,128,0.008898133039474487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,256,0.01843199928601583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,128,0.020522665977478028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,4096,0.07529493172963461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,64,0.007706666489442189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,32,0.008056533336639405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,64,0.02072426676750183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,32,0.02079253395398458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,51200,1.0054709116617837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,2560,0.058594131469726564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,768,0.021639466285705566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,256,0.02228053410847982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,65536,1.1476266225179037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,65536,1.3863189697265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,51200,1.0888810475667319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,65536,1.2018826802571616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,16384,0.32714665730794273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,12288,0.28044373194376626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,16384,0.317411200205485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,16384,0.32152961095174154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,51200,0.8906080245971679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,12288,0.2375648021697998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,10240,0.21976426442464195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,10240,0.19926613171895344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,10240,0.17673707008361816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,8192,0.1653216044108073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,8192,0.17115306854248047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,7168,0.14380480448404948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,7168,0.14015040397644044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,6144,0.12430400053660076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,6144,0.12108266353607178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,5120,0.10548799832661945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,768,0.025608533620834352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,5120,0.10281706651051839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,5120,0.09013013044993082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,128,0.016450132926305136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,4096,0.08460586865743001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,3584,0.07484373251597086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,3584,0.07612693309783936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,4096,0.08522559801737467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,3072,0.06512639919916788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,3072,0.06787199974060058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,12288,0.20678720474243165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,2560,0.05494720141092936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,51200,0.8897610982259115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,2560,0.05927573442459107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,8192,0.14095147450764972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,2048,0.04582506815592448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,7168,0.1271456003189087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,2048,0.04954346815745036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,2048,0.04176853497823079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,6144,0.1056010643641154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,1536,0.04102079868316651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,1024,0.02609386642773946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,1024,0.033336532115936277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,1024,0.026778666178385417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,768,0.021383466323216756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,768,0.02866986592610677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,4096,0.0736255963643392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,512,0.01646080017089844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,512,0.02550613284111023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,3584,0.0649237314860026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,256,0.011351466178894043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,256,0.021993599335352578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,1536,0.035651199022928876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,128,0.008524800340334576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,128,0.020439465840657554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,3072,0.05695039828618368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,64,0.007361066838105519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,64,0.020833067099253335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,32,0.007894399762153625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,32,0.021000534296035767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,2560,0.05068693161010742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,1536,0.03282666603724162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,65536,1.14650026957194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,768,0.02141973376274109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,65536,1.0178858439127603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,65536,0.8717130661010742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,51200,0.8345290501912436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,512,0.017593600352605186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,16384,0.26105066935221355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,256,0.014478933811187745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,51200,0.6796693166097005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,12288,0.19672853151957195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,16384,0.2953109423319499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,12288,0.21279679934183754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,16384,0.25790294011433923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,51200,0.8386997222900391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,12288,0.16373119354248047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,10240,0.18429333368937176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,8192,0.13309333324432374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,10240,0.15913813908894855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,8192,0.15905067125956218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,7168,0.11641919612884521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,8192,0.12010453542073567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,6144,0.10026559829711915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,7168,0.13837119738260906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,128,0.012426666418711345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,5120,0.0856223980585734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,6144,0.12280320326487223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,5120,0.09459199905395507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,5120,0.0878048022588094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,4096,0.06911466916402181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,4096,0.07922240098317465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,3584,0.059995734691619874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,3584,0.054484268029530845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,3072,0.05317013263702393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,3072,0.06191573143005371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,2560,0.045664000511169436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,3072,0.04842880169550578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,2048,0.036400000254313156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,2560,0.05279146830240885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,2048,0.037435734272003175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,2048,0.043722665309906004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,1536,0.028433066606521607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,1536,0.03643946647644043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,1536,0.031274666388829545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,1024,0.020338133970896403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,1024,0.02999146580696106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,1024,0.023856000105539957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,768,0.01733760039011637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,768,0.026105600595474242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,768,0.021319466829299926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,512,0.01320319970448812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,512,0.023459200064341226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,512,0.017485866943995156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,256,0.009194667140642803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,256,0.015375999609629312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,256,0.021332265933354695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,128,0.007335466643174489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,128,0.019062399864196777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,128,0.013877333203951518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,64,0.006400000055631001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,32,0.006738133231798808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,64,0.019242666165033975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,32,0.019128533204396565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,3584,0.07438399791717529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,65536,0.8376202901204428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,65536,0.9172106424967448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,10240,0.17014613151550292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,65536,0.9377877553304037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,51200,0.7181237538655598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,16384,0.231933863957723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,7168,0.09983999729156494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,51200,0.7237333297729492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,12288,0.15744746526082357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,6144,0.08650666872660319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,16384,0.26487785975138345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,4096,0.06085973183314005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,12288,0.1854090690612793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,10240,0.1361962636311849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,8192,0.10946986675262452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,10240,0.15647145907084148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,8192,0.132040532430013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,8192,0.11536426544189453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,7168,0.09389440218607584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,7168,0.11230826377868652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,7168,0.10262506802876789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,6144,0.08221333026885987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,6144,0.09852586587270101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,6144,0.08666986624399821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,5120,0.06824213663736979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,5120,0.08294080098470052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,4096,0.056357332070668543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,4096,0.06981653372446696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,4096,0.06021866798400879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,3584,0.04990826845169068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,3584,0.06277973254521688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,2560,0.042209064960479735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,3584,0.05321386655171713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,3072,0.04319893519083659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,3072,0.05480320056279501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,3072,0.045739734172821046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,2560,0.03662293354670207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,2560,0.0468394676844279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,2048,0.030080000559488933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,2048,0.039741865793863934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,5120,0.07561600208282471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,1536,0.0240938663482666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,1536,0.03373866478602092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,1536,0.02701866626739502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,1024,0.01797973314921061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,1024,0.02845333417256673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,16384,0.22274239857991537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,1024,0.021913599967956544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,12288,0.16876373291015626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,768,0.015094400445620219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,768,0.02523626685142517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,512,0.01162986656030019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,512,0.022259199619293214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,51200,0.6895061492919922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,256,0.008520533641179402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,256,0.02027626633644104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,10240,0.1489674727121989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,128,0.007011199990908305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,128,0.018950400749842326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,128,0.011244799693425496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,64,0.00618453323841095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,32,0.0064735998709996535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,64,0.019492266575495402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,32,0.018733867009480796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,65536,0.6721247990926107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,65536,0.6529578526814779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,65536,0.8319616317749023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,512,0.014114133516947427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,51200,0.5521408081054687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,16384,0.17015679677327472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,51200,0.6603424072265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,51200,0.5015040079752604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,2560,0.04083520174026489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,16384,0.23454720179239907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,12288,0.12779413064320883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,10240,0.10671359697977703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,2048,0.033580799897511796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,10240,0.14357226689656574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,8192,0.08590293725331624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,8192,0.11708693504333496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,8192,0.08566719690958659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,7168,0.0760095993677775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,7168,0.10199999809265137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,6144,0.06557546854019165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,6144,0.08842346668243409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,6144,0.06499733527501425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,768,0.01726613243420919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,5120,0.05522346496582031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,5120,0.07531733512878418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,4096,0.044495999813079834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,256,0.011646933356920878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,4096,0.062249600887298584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,4096,0.04616213242212931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,3584,0.03947946627934774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,3584,0.054571731885274255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,3584,0.04175999959309896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,12288,0.1690399964650472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,3072,0.033973332246144614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,2560,0.028885332743326823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,3072,0.047713065147399904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,2560,0.03176533381144206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,2048,0.024411733945210776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,2048,0.035495467980702716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,2048,0.027663999795913698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,1536,0.019729065895080566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,7168,0.07609919706980386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,1536,0.02318613330523173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,1536,0.030074665943781536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,1024,0.014686933159828186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,1024,0.02547946572303772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,1024,0.018427733580271402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,768,0.012330666184425354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,768,0.023000532388687135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,16384,0.1669578711191813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,512,0.00983679989973704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,2560,0.041152000427246094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,512,0.021129600207010903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,256,0.007529599964618683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,12288,0.1252789338429769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,256,0.01908479928970337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,128,0.0062730665008227035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,10240,0.10596586863199871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,128,0.010831999778747558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,64,0.005357866485913595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,64,0.01816533406575521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,32,0.0056981335083643595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,32,0.017754666010538735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,5120,0.05560533205668131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,3072,0.037281068166097005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,128,0.018068265914916993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,65536,0.752945073445638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,51200,0.40941654841105146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,16384,0.13338133494059246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,768,0.016199466586112977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,65536,0.5112693468729655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,16384,0.20898772875467936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,12288,0.09792746702829996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,51200,0.46206188201904297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,10240,0.08227519989013672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,12288,0.1571381409962972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,10240,0.13206079800923665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,10240,0.10075626373291016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,8192,0.06631040175755819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,8192,0.10812906424204509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,512,0.01376426617304484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,7168,0.059944534301757814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,256,0.011747200290362041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,7168,0.09274773597717285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,6144,0.051130668322245276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,6144,0.08065386613210043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,5120,0.04233173529307048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,5120,0.06779306729634603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,51200,0.5920415878295898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,4096,0.03509653409322103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,4096,0.05633066495259603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,65536,0.5813333511352539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,3584,0.030714666843414305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,3584,0.04854186773300171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,16384,0.15227306683858235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,3072,0.02704213261604309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,3072,0.042293334007263185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,12288,0.11522666613260905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,2560,0.023033599058787026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,2560,0.037563733259836835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,8192,0.08022293249766031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,2048,0.01949119965235392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,2048,0.032603732744852704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,7168,0.07038613160451254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,1536,0.015964800119400026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,6144,0.05880000193913778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,1536,0.027926399310429888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,1024,0.011730133493741354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,1024,0.023717333873113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,5120,0.051744000116984046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,768,0.009718400239944459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,768,0.02140480081240336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,4096,0.0432096004486084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,512,0.007897600034872691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,512,0.02050559918085734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,3584,0.038471468289693195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,256,0.00617386649052302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,256,0.018351999918619792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,3072,0.03331306576728821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,128,0.005194666484991709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,128,0.017403733730316163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,2560,0.0291157325108846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,64,0.004694400231043497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,64,0.01717653274536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,32,0.004914133250713349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,2048,0.024949334065119424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,32,0.017112533251444496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,1536,0.019615999857584634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,1024,0.016101333498954772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,65536,0.3401631991068522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,768,0.013587199648221335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,65536,0.42683839797973633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,51200,0.2757610638936361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,65536,0.6632821400960286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,16384,0.08969813187917074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,16384,0.11309759616851807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,12288,0.06770239671071371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,16384,0.17682453791300456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,51200,0.3310720125834147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,51200,0.5313536008199056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,10240,0.05761066675186157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,12288,0.08639360268910726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,12288,0.13510185877482098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,10240,0.11556800206502278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,8192,0.04639039834340413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,512,0.011891200145085653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,7168,0.0397866686185201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,8192,0.09509332974751791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,7168,0.08412906328837076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,6144,0.03522026538848877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,7168,0.05186026493708292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,5120,0.02927253246307373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,6144,0.045764267444610596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,6144,0.07229866981506347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,5120,0.03981333176294963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,5120,0.05920639832814535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,4096,0.047278932730356854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,3584,0.021233065923055013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,3584,0.04164906740188599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,256,0.01009493370850881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,8192,0.05827946662902832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,3072,0.018425599733988444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,3072,0.036765865484873456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,2560,0.016270933548609416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,2560,0.033090132474899295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,4096,0.024493867158889772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,128,0.00867306689421336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,2048,0.013860266407330832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,4096,0.03262613415718078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,2048,0.029128533601760865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,1536,0.011062399546305338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,1536,0.025618133942286174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,1536,0.016075733304023742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,1024,0.008539733290672303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,1024,0.02187946637471517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,768,0.007430399954319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,768,0.02000746726989746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,768,0.011787733435630799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,512,0.0062047998110453285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,512,0.01941546599070231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,10240,0.0721781333287557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,256,0.005190399785836538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,256,0.017246933778127034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,128,0.004409599800904592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,128,0.01655359963575999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,3584,0.029386667410532634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,64,0.00403413325548172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,64,0.01639359990755717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,32,0.004312533140182495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,32,0.016484266519546507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,65536,0.26959358851114906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,3072,0.025781333446502686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,2048,0.019530665874481202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,2560,0.02256960074106852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,65536,0.6449717203776042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,65536,0.4037215868631999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,51200,0.2154698689778646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,1024,0.013118933637936911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,16384,0.07204480171203613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,12288,0.057061334451039634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,16384,0.1669834613800049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,51200,0.5075551986694335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,512,0.010135466853777569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,10240,0.049737600485483806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,12288,0.0822218656539917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,8192,0.03652160167694092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,10240,0.11299946308135986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,8192,0.09075626532236734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,7168,0.03272639910380046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,7168,0.08015147050221762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,256,0.009136000275611877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,6144,0.029838933547337847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,6144,0.06908373037974039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,128,0.00791786660750707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,5120,0.026504532496134443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,12288,0.13344426155090333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,5120,0.038073599338531494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,5120,0.05597546497980753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,4096,0.019803732633590698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,4096,0.04442773262659709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,3584,0.01732053359349569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,3584,0.03869013388951619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,16384,0.10712426503499348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,3072,0.016270933548609416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,51200,0.3240864117940267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,3072,0.03463360071182251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,2560,0.013691733280817667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,8192,0.05539520184199015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,2560,0.021130667130152384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,2560,0.030430932839711506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,2048,0.012107732892036437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,2048,0.02661866744359334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,1536,0.009823999802271525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,7168,0.050078932444254556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,1536,0.02478826642036438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,1024,0.007281066477298736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,1024,0.020869332551956176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,1024,0.011911466717720032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,768,0.006428800026575724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,6144,0.04313173294067383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,768,0.019314134120941163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,512,0.0054848000407218935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,512,0.01856000026067098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,512,0.009305600325266521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,256,0.004598399996757508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,256,0.016898133357365928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,10240,0.06777173678080241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,4096,0.03072106639544169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,256,0.007852800190448761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,128,0.01613759994506836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,3584,0.027430399258931475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,128,0.00692799985408783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,64,0.0037151999771595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,32,0.0038506666819254553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,64,0.015923200050989787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,32,0.016057599584261575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,65536,0.1827445348103841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,65536,0.5919093449910482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,51200,0.15159786542256673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,128,0.004002133260170618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,51200,0.4634592056274414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,3072,0.02387839953104655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,16384,0.05256213347117106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,2048,0.01789439916610718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,12288,0.0496394673983256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,1536,0.014855466286341348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,16384,0.16237012545267743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,10240,0.04287039836247762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,12288,0.12702293395996095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,10240,0.10960426330566406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,8192,0.02993173400561015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,768,0.010360532999038696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,8192,0.08805226484934489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,7168,0.02924693425496419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,7168,0.07701333363850912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,6144,0.025937066475550337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,6144,0.06531413396199545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,65536,0.35998080571492513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,5120,0.02281493345896403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,5120,0.0531712015469869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,51200,0.2874538739522298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,4096,0.01739306648572286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,16384,0.09623040358225504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,12288,0.0742037296295166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,4096,0.041706665356953936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,3584,0.01597546637058258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,3584,0.03699839909871419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,3072,0.01453013320763906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,10240,0.06284159819285075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,3072,0.033089067538579306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,2560,0.01204800009727478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,8192,0.05067413250605265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,2560,0.029725867509841918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,2048,0.010273067156473796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,2048,0.026741333802541095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,7168,0.04456319808959961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,1536,0.0083146666487058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,1536,0.02314773400624593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,6144,0.03910826841990153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,1024,0.006437333424886067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,1024,0.02016213337580363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,5120,0.03364799817403157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,768,0.0055754666527112326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,768,0.019228800137837728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,4096,0.027221333980560303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,512,0.004762666424115499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,512,0.018097066879272462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,3072,0.021465599536895752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,256,0.004127999891837438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,256,0.016568533579508462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,2048,0.016687999169031777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,128,0.0036864000062147772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,128,0.015706666310628257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,3584,0.02467093269030253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,64,0.003386666625738144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,64,0.015685333808263143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,32,0.0035135999321937563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,32,0.016011733810106912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,65536,0.11733226776123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,65536,0.5551317214965821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,1024,0.011100799838701884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,51200,0.09453866481781006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,2560,0.01934079925219218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,1536,0.013868799805641175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,16384,0.03698346614837646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,51200,0.44075625737508134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,16384,0.1532021363576253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,12288,0.03306559920310974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,12288,0.11971413294474284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,256,0.007659733295440674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,10240,0.029603199164072676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,512,0.008610133330027263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,10240,0.10211946964263915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,128,0.006908800204594929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,8192,0.021249065796534218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,768,0.009612799684206644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,8192,0.08195093472798666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,7168,0.01978773276011149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,7168,0.07114666302998861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,6144,0.01664426624774933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,6144,0.06016319990158081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,65536,0.33458239237467446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,5120,0.015675733486811318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,5120,0.04725653330485026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,51200,0.2621023972829183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,4096,0.010939733187357584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,16384,0.08728746573130289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,4096,0.03728533188501994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,12288,0.06673493385314941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,3584,0.009827199578285217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,3584,0.0330186665058136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,10240,0.0571616013844808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,3072,0.009369599819183349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,8192,0.04592959880828858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,3072,0.03036373257637024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,7168,0.041042133172353105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,2560,0.0077898666262626644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,2560,0.028624000151952107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,6144,0.036652799447377524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,2048,0.006646400193373363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,2048,0.025722666581471758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,1536,0.006040533383687338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,1536,0.022403200467427574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,1024,0.004805333415667216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,1024,0.020300799608230592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,5120,0.030444800853729248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,768,0.00450133333603541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,768,0.01853653391202291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,4096,0.02458453377087911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,512,0.004004266609748204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,512,0.017501866817474364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,3584,0.02233920097351074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,256,0.0035477332770824433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,256,0.01616320013999939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,3072,0.01958613395690918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,128,0.003223466624816259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,128,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,1024,0.009874133268992107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,2048,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,64,0.0032127998769283296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,32,0.0033429334561030067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,64,0.015362133582433065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,32,0.015125333269437154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,65536,0.10778240362803142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,2560,0.017577600479125977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,65536,0.5500586827596028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,51200,0.08562133312225342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,1536,0.01258666714032491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,16384,0.03235413432121277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,16384,0.14892479578653972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,51200,0.43295148213704426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,12288,0.02550826668739319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,12288,0.11538346608479817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,768,0.008740267157554627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,10240,0.022175999482472737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,10240,0.09886826674143473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,512,0.00767573316891988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,8192,0.013413332899411521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,8192,0.07883413632710776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,256,0.006877866884072621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,7168,0.012200533350308736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,7168,0.06869226296742757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,128,0.005973333120346069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,6144,0.015225600202878317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,6144,0.056850131352742514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,65536,0.31965548197428384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,5120,0.014007467031478881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,5120,0.044487468401590985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,51200,0.254911994934082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,16384,0.08553600311279297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,4096,0.010868266224861145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,4096,0.03605440060297648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,3584,0.0100490669409434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,3584,0.03277653257052104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,10240,0.054656000932057705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,3072,0.009186133742332458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,3072,0.029458133379618327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,12288,0.06520533164342245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,2560,0.007213866710662842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,2560,0.02675093412399292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,8192,0.04480533202489217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,2048,0.006345599889755249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,2048,0.024471465746561685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,7168,0.03975253502527873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,1536,0.005588266750176748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,1536,0.02220053275426229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,6144,0.03437973260879516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,1024,0.004676266511281332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,1024,0.019483733177185058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,5120,0.029284266630808513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,768,0.004223999877770742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,768,0.018114133675893148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,4096,0.023779199520746867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,512,0.003764266769091288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,512,0.017178666591644288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,3584,0.02182933290799459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,256,0.003293866664171219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,256,0.0162090669075648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,3072,0.018984532356262206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,128,0.00317546675602595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,128,0.015270400047302245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,2560,0.01685333251953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,64,0.0029919999341169994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,64,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,32,0.003050666550795237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,32,0.015012266238530478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,2048,0.014295466740926108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,65536,0.10380480289459229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,1024,0.009598933657010396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,51200,0.07965439955393473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,1536,0.01213759978612264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,16384,0.029360000292460126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,65536,0.5449269612630208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,12288,0.023745065927505492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,16384,0.14744213422139485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,10240,0.019665066401163736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,12288,0.1134218692779541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,51200,0.42895145416259767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,8192,0.01138879954814911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,10240,0.09637013276418051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,7168,0.009708799918492635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,8192,0.07617493470509848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,6144,0.015125333269437154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,7168,0.0670250654220581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,5120,0.008718933661778767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,6144,0.05570240020751953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,4096,0.007835733393828075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,5120,0.043668266137441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,3584,0.007530666887760162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,3072,0.006963199873765309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,3584,0.03140906691551208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,2560,0.006460799773534138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,3072,0.02877333362897237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,2048,0.005727999905745188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,2560,0.02611306707064311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,1536,0.00499839981396993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,2048,0.02400426665941874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,1024,0.004281599819660187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,1536,0.021347200870513915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,768,0.003942399968703588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,1024,0.019197867314020792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,768,0.017853866020838417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,512,0.017324799299240114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,256,0.0032074667513370516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,256,0.015938133001327515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,768,0.008296533425649007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,128,0.0029440000653266907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,64,0.0028223998844623564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,128,0.015434666474660238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,32,0.0028586665789286296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,64,0.014882133404413859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,32,0.015030399958292643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,65536,0.09940160115559896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,51200,0.07711466948191324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,512,0.007396266857783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,16384,0.02823999921480815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,4096,0.03513706525166829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,65536,0.5425578435262044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,16384,0.1460426648457845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,51200,0.4266965230305989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,10240,0.015441067020098367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,12288,0.02271359960238139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,8192,0.0098880002895991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,12288,0.1120416005452474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,7168,0.009274666508038838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,6144,0.00844266712665558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,8192,0.0753333330154419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,5120,0.007725866635640462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,7168,0.06534399986267089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,4096,0.0070271998643875126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,6144,0.05370239814122518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,3584,0.006769066552321117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,256,0.0066442668437957765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,5120,0.04336746533711751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,4096,0.03487786849339803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,3072,0.006400000055631001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,2560,0.0061258668700853985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,128,0.005895466605822245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,3072,0.02876480023066203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,2048,0.0054666668176651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,2560,0.026305067539215087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,1024,0.00425493319829305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,2048,0.023749333620071412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,768,0.0038890667259693147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,1536,0.02142933408419291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,512,0.003554133325815201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,768,0.01844373345375061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,256,0.0032320000231266023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,512,0.016775466998418174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,128,0.003052799900372823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,256,0.01600213348865509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,64,0.0031445334355036415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,128,0.015396266182263692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,32,0.0029685333371162414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,64,0.014898133277893067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,32,0.014844800035158793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,10240,0.0949184020360311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,512,0.00365226666132609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,3584,0.03118293285369873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,1536,0.004845866560935974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,1024,0.019321600596110024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,12288,1.9630762736002605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,16384,2.9546539306640627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,12288,3.9160074869791663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,16384,5.190641276041666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,12288,2.8530901590983073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,10240,1.7877194722493488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,10240,3.451171112060547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,16384,4.065020751953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,8192,1.4327786763509116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,51200,8.712225341796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,8192,2.7393419901529947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,8192,1.8686154683430989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,7168,1.237175496419271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,7168,2.3146133422851562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,6144,1.0804747263590495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,7168,1.7036394755045574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,6144,1.4672757466634114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,6144,2.010205841064453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,5120,0.8914335886637369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,5120,1.192936579386393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,4096,0.6887189229329427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,5120,1.7365408579508466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,4096,0.9457663853963216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,4096,1.6026165008544921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,3584,0.6363701502482096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,3584,0.899787712097168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,3584,1.3771690368652343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,3072,0.5315679868062337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,3072,0.7021130879720052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,3072,0.9831146876017252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,2560,0.5655498504638672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,2560,0.9378559748331705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,2560,0.6120586395263672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,2048,0.40888427098592117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,2048,0.6681610743204753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,1536,0.3120565414428711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,10240,2.34728520711263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,1536,0.55676695505778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,2048,0.6438709259033203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,1536,0.42664000193277996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,1024,0.2327295939127604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,768,0.20286186536153158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,768,0.3055466651916504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,768,0.2947253227233887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,1024,0.367796262105306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,51200,17.850948079427084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,512,0.14222505887349446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,256,0.1155733346939087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,51200,17.597135416666667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,128,0.10537066459655761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,256,0.1204416036605835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,512,0.24571839968363443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,128,0.10883519649505616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,64,0.0956928014755249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,32,0.09453012943267822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,32,0.10694719950358074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,256,0.20555200576782226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,65536,128,0.2264959971110026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,1024,0.35949652989705405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,65536,512,0.19699093500773113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,65536,64,0.10864853064219157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,16384,2.2994112650553387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,16384,4.124079895019531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,16384,2.933655548095703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,51200,6.958841451009114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,12288,3.0648778279622393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,12288,1.6554815928141278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,65536,8.828530883789062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,12288,2.230791473388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,10240,1.4208501180013022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,10240,2.6499755859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,10240,1.9929088592529296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,8192,1.1188458760579427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,51200,11.680279541015626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,8192,2.115449651082357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,8192,1.4598336537679037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,7168,0.9505557378133138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,51200,13.75361328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,6144,1.0106784184773763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,7168,1.508676274617513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,7168,1.9986250559488934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,6144,1.8245066324869792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,6144,1.09662717183431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,5120,0.8466837565104166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,5120,1.06123841603597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,4096,0.6230752309163411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,4096,1.1821493784586588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,5120,1.367682139078776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,4096,0.8496831893920899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,3584,0.5240832010904948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,3584,1.0532447814941406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,3584,0.6201834360758464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,3072,0.4347850799560547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,3072,0.6200650533040364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,3072,0.8266261418660482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,2560,0.40761388142903643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,2560,0.47370026906331375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,2048,0.5281503995259602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,2560,0.7045194625854492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,2048,0.30958080291748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,1536,0.2549023946126302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,1536,0.4168586730957031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,2048,0.47205333709716796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,1024,0.18852799733479816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,1024,0.3143978754679362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,1536,0.3542133331298828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,1024,0.2678197224934896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,768,0.23022079467773438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,768,0.22551466623942057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,512,0.16222292582194012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,512,0.12580373287200927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,512,0.21244799296061195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,256,0.10688213507334392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,65536,18.214892578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,256,0.1556106726328532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,256,0.11536426544189453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,128,0.09373653729756673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,65536,18.18060302734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,64,0.0801098664601644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,32,0.07889920075734456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,64,0.09286613464355468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,32,0.10119040012359619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,51200,128,0.17082346280415853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,51200,768,0.16364266077677408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,51200,128,0.08892160256703695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,16384,1.2135818481445313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,51200,2.465476226806641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,16384,0.7614037195841472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,65536,3.0593409220377605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,16384,0.9007786432902017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,12288,0.5846730550130208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,51200,3.1130388895670573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,12288,0.9716597239176432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,12288,0.6897258758544922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,10240,0.8505205154418946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,65536,4.190397898356119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,10240,0.48115841547648114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,8192,0.40067307154337567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,10240,0.6238026936848958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,8192,0.6618026733398438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,7168,0.3362026532491048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,65536,5.103656514485677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,8192,0.46996587117513017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,7168,0.5738367716471354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,7168,0.40693333943684895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,6144,0.3050890604654948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,6144,0.34807039896647135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,6144,0.5055626551310222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,5120,0.2594186623891195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,5120,0.4065216064453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,4096,0.21622079213460288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,5120,0.30390612284342444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,4096,0.32391573588053385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,4096,0.25626773834228517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,3584,0.18558506965637206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,3584,0.29573332468668617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,3072,0.16585920651753744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,3072,0.24011626243591308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,3072,0.19972373644510905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,2560,0.14003626505533856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,2560,0.20720319747924804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,51200,4.022484334309896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,2560,0.17255466779073078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,2048,0.16012694040934244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,2048,0.1256170670191447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,1536,0.12431786855061848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,2048,0.14570879936218262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,1536,0.10050666332244873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,1024,0.09603306452433268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,768,0.07209920088450114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,512,0.054223998387654626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,1024,0.07321600119272867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,768,0.06395413478215536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,512,0.05221226612726847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,256,0.0360149343808492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,256,0.04412586688995361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,512,0.06775039831797282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,128,0.030061866839726763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,256,0.05645333528518677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,64,0.02773759961128235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,128,0.05085440079371134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,64,0.04201600154240926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,16384,32,0.028871466716130573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,32,0.04323413372039795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,3584,0.22672425905863441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,1536,0.10820586681365967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,768,0.08753173351287842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,16384,128,0.04167360067367554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,16384,1.0123317082722982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,65536,2.5444798787434895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,16384,0.617795181274414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,51200,1.9245428721110027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,65536,3.108600616455078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,51200,2.1784289042154947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,16384,0.6738901138305664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,12288,0.46462825139363606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,12288,0.754313596089681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,51200,3.287108357747396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,12288,0.5606271743774414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,10240,0.4220394770304362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,65536,4.142762756347656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,10240,0.6844181060791016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,10240,0.4322965304056804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,8192,0.32072426478068033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,8192,0.36230827967325846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,8192,0.513972282409668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,7168,0.4435733477274577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,7168,0.3219402631123861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,6144,0.38316265741984046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,6144,0.24587732950846353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,6144,0.28109973271687827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,5120,0.334985605875651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,5120,0.215283203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,4096,0.24097065925598143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,4096,0.18119680086771647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,4096,0.1959712028503418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,3584,0.21138453483581543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,3584,0.15774720509847004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,3584,0.1687594731648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,3072,0.18957759539286295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,3072,0.13848533630371093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,3072,0.14553279876708985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,2560,0.15425920486450195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,2560,0.12383039792378743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,2560,0.12588799794514974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,2048,0.13286933104197185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,2048,0.10170453389485676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,1536,0.09660693009694418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,2048,0.1115114688873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,1536,0.08130346934000651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,1536,0.08523733615875244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,1024,0.06749227046966552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,1024,0.06502506732940674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,768,0.055493334929148354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,1024,0.07087893486022949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,768,0.054681599140167236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,512,0.044514131546020505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,768,0.061513598759969076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,512,0.04564053217569987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,512,0.04980159997940063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,256,0.030406399567921953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,128,0.02472426692644755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,256,0.03824106852213542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,7168,0.2866410573323568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,128,0.036160000165303546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,128,0.038734932740529374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,64,0.02294293244679769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,12288,32,0.02225386699040731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,64,0.036722131570180255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,12288,32,0.03675093253453572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,5120,0.24147307078043617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,12288,256,0.042634665966033936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,51200,1.599277877807617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,65536,2.106892776489258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,16384,0.5156650543212891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,16384,0.8723199844360352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,65536,3.3016352335611976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,12288,0.6165461222330729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,16384,1024,0.08599786758422852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,12288,0.4138613382975261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,10240,0.333135986328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,10240,0.5310453414916992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,51200,2.758477783203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,8192,0.4050314585367839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,8192,0.3056447982788086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,7168,0.25283625920613606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,7168,0.36880852381388346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,6144,0.31875200271606446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,6144,0.222162135442098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,12288,0.5829685211181641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,5120,0.2468565305074056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,5120,0.17421654065450032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,16384,0.7876352310180664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,4096,0.20229120254516603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,4096,0.1499733289082845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,6144,0.2951626777648926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,10240,0.48926506042480467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,3584,0.18072959582010906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,3584,0.12668800354003906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,3072,0.1545962651570638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,3072,0.113811198870341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,65536,4.625480651855469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,2560,0.1347775936126709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,2560,0.10267626444498698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,7168,0.3519925435384115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,2048,0.1068341334660848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,2048,0.08398719628651938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,8192,0.3952255884806315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,51200,3.367055002848307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,1536,0.08238293329874674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,1536,0.07503360112508138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,1024,0.06404800017674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,1024,0.057418668270111085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,3072,0.1469183921813965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,768,0.04728533426920573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,768,0.047953065236409506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,1536,0.07552106380462646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,4096,0.2010709285736084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,512,0.03608959913253784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,512,0.04054400126139323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,256,0.02571413318316142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,256,0.033369600772857666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,5120,0.2470421314239502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,128,0.022362667322158813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,128,0.03115306695302327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,2560,0.12697813510894776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,3584,0.17413652737935384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,64,0.017744000752766928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,64,0.03130773305892944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,10240,32,0.018272000551223754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,10240,32,0.03150186737378438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,1024,0.05897706747055054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,2048,0.10141332944234212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,65536,1.629421869913737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,65536,2.684778594970703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,51200,2.0549888610839844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,256,0.031041065851847332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,51200,1.3411946614583334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,65536,3.3993301391601562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,16384,0.421399466196696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,16384,0.6047232309977214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,128,0.026187733809153242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,512,0.0366154670715332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,12288,0.4672671953837077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,12288,0.32099841435750326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,16384,0.652565320332845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,10240,0.36950292587280276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,10240,0.2690314610799154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,8192,0.25207680066426597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,8192,0.30114558537801106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,10240,0.40274667739868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,7168,0.2605151971181234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,7168,0.21319252649943032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,6144,0.23126506805419922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,6144,0.16820054054260253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,5120,0.1938752015431722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,5120,0.14557654062906902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,4096,0.1533674716949463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,4096,0.12146026293436687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,10240,768,0.04552106857299805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,3584,0.136407470703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,3584,0.10699413617451985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,12288,0.46895999908447267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,51200,2.315451812744141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,3072,0.11884693304697673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,8192,0.32043412526448567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,3072,0.11199573675791423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,2560,0.10012693405151367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,2560,0.08869120279947916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,7168,0.2824106534322103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,2048,0.08745813369750977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,2048,0.07185599803924561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,6144,0.24041172663370766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,1536,0.0642410675684611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,1536,0.06037439902623495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,1024,0.046845865249633786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,5120,0.19783466657002766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,1024,0.048254934946695964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,768,0.03811946709950765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,4096,0.16302186648050945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,768,0.042132266362508136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,512,0.029381332794825236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,512,0.03622399965922038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,3584,0.14091946283976237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,256,0.020396800835927327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,256,0.03026240070660909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,3072,0.11748692989349366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,128,0.01671253244082133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,128,0.027914667129516603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,2560,0.1036725362141927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,64,0.014074666301409402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,64,0.028459733724594115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,2048,0.08268799781799316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,8192,32,0.015077333648999533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,1536,0.06122453212738037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,8192,32,0.02763200004895528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,1024,0.04877866506576538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,768,0.03814613421758016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,512,0.031175466378529866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,51200,1.1967243194580077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,65536,1.5510218302408854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,256,0.026374399662017822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,51200,1.8389631907145183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,16384,0.5531477610270182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,65536,2.3761492411295575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,12288,0.31678826014200845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,12288,0.46232961018880203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,16384,0.5666378657023112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,8192,128,0.022265599171320597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,10240,0.34897066752115885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,10240,0.26957546869913734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,8192,0.2673386573791504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,8192,0.21937813758850097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,7168,0.18688534100850424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,7168,0.24037013053894044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,16384,0.4026805241902669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,6144,0.21202133496602377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,7168,0.2527989387512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,6144,0.1561386744181315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,5120,0.17211519877115886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,5120,0.1337824026743571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,5120,0.17660160064697267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,4096,0.1414613405863444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,4096,0.11424960295359295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,4096,0.1453162670135498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,3584,0.12318507035573323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,3584,0.10067306359608967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,3584,0.12456320126851399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,3072,0.10663786729176838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,3072,0.08947626749674478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,51200,1.9127541859944661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,2560,0.1118282636006673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,12288,0.416871452331543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,2560,0.08819093704223632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,2048,0.0847050666809082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,10240,0.34794772466023766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,2048,0.07085119883219401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,2048,0.07419306437174479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,1536,0.06123840014139811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,1536,0.055958398183186854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,8192,0.27967678705851234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,1536,0.05624106725056967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,1024,0.042071465651194254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,1024,0.04444479942321777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,768,0.034635734558105466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,768,0.03853546778361003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,512,0.026794666051864625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,512,0.03562560081481934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,6144,0.20873173077901205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,256,0.020278400182723998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,256,0.030033065875371294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,65536,2.8556287129720053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,128,0.015677866339683533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,128,0.026467200120290118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,64,0.01307093302408854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,64,0.026717867453893023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,7168,32,0.013470932841300964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,7168,32,0.02664426763852437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,3072,0.10494186878204345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,2560,0.0921183983484904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,65536,1.3624820709228516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,1024,0.04406079848607381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,768,0.034728535016377765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,65536,1.9499487559000652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,512,0.028498133023579914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,256,0.02377386689186096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,16384,0.35182078679402673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,16384,0.4874143918355306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,16384,0.4961386680603027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,51200,1.0944341023763022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,12288,0.34512958526611326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,12288,0.28930025100708007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,7168,128,0.020152533054351808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,10240,0.24029547373453775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,51200,1.5781035105387369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,10240,0.29672746658325194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,8192,0.2301568031311035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,8192,0.1948362668355306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,7168,0.20967253049214682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,7168,0.1601141293843587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,8192,0.253654400507609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,65536,2.4194516499837238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,6144,0.18136746088663738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,6144,0.13931627273559571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,7168,0.22156906127929688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,6144,0.18562347094217937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,4096,0.12058773040771484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,5120,0.1324501355489095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,5120,0.16675200462341308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,4096,0.10404373009999593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,3584,0.11485546429951984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,3584,0.09916053613026937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,3072,0.09509332974751791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,3072,0.08862826824188233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,2560,0.08018240133921305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,2560,0.07484052975972494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,2560,0.08104960123697916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,2048,0.06406399806340536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,2048,0.0626581350962321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,2048,0.06353813409805298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,1536,0.05034453471501669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,1536,0.052559999624888096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,12288,0.3561951955159505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,1024,0.03625919818878174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,1024,0.04143786827723185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,1024,0.03906453450520833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,768,0.029908267656962077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,768,0.036083201567331954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,51200,1.6638015747070312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,512,0.02934719920158386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,512,0.03887573480606079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,5120,0.1530794620513916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,256,0.016641066471735636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,3584,0.10638079643249512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,256,0.02618880073229472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,4096,0.12814826965332032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,128,0.012859732906023661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,3072,0.09161279996236166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,128,0.025217066208521526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,64,0.011553066968917846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,64,0.026130133867263795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,6144,32,0.026215465863545735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,10240,0.30264533360799156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,1536,0.04779200156529744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,6144,32,0.01353600025177002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,65536,1.2158111572265624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,65536,1.7063296000162762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,768,0.030834132432937623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,512,0.02515946626663208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,51200,0.9696640014648438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,51200,1.3119796752929687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,256,0.021191465854644775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,16384,0.3809845288594564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,6144,128,0.017682133118311565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,12288,0.2836341222127279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,16384,0.32595841089884436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,12288,0.2578218619028727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,10240,0.24134507179260253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,12288,0.30434773763020834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,8192,0.19503787358601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,10240,0.20576853752136232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,7168,0.16989653905232746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,8192,0.16004692713419597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,7168,0.14168853759765626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,6144,0.145142396291097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,7168,0.18730559349060058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,6144,0.12216533025105794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,5120,0.12326506773630778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,5120,0.105021866162618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,5120,0.13234346707661945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,4096,0.10091839631398518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,4096,0.0874229351679484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,4096,0.10872426827748616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,3584,0.08830506801605224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,3584,0.08085333506266276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,3072,0.0773301362991333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,65536,2.0332661946614583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,3072,0.07238399982452393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,2560,0.06547626654307047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,2560,0.06320319970448812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,16384,0.413099733988444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,2048,0.0538271983464559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,2048,0.055490132172902426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,2048,0.055978667736053464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,10240,0.25781760215759275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,1536,0.04176533222198486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,1536,0.04251840114593506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,51200,1.4725205739339193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,1024,0.03004266619682312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,1024,0.038355199495951335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,1024,0.0344874660174052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,768,0.02667306661605835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,768,0.031242666641871135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,512,0.018730666240056357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,512,0.02765226761500041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,512,0.022207999229431154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,256,0.012966400384902954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,256,0.023290665944417317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,1536,0.04389546712239583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,8192,0.20900373458862304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,128,0.00978773335615794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,128,0.015501866738001505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,128,0.025081600745519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,64,0.0084906667470932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,6144,0.15490773518880208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,5120,32,0.011565867066383361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,64,0.02395413319269816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,5120,32,0.024284799893697105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,3584,0.09438506762186685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,3072,0.08015680313110352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,2560,0.0701578696568807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,65536,1.248375447591146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,51200,1.0162229537963867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,16384,0.33799680074055993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,768,0.026970666646957398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,51200,0.8159253438313803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,12288,0.22397866249084472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,12288,0.20760107040405273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,16384,0.3077695846557617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,10240,0.19017173449198405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,5120,256,0.018127999703089395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,10240,0.19937599500020345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,8192,0.15371947288513182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,8192,0.1449834664662679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,65536,1.0431626637776694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,7168,0.13632853825887042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,8192,0.171235195795695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,7168,0.12246613502502442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,6144,0.1166805346806844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,7168,0.15564907391866048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,6144,0.10762879848480225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,5120,0.09915093580881754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,5120,0.09349119663238525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,4096,0.07867519855499268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,4096,0.07847999731699626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,10240,0.21470826466878257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,3584,0.06963093280792236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,3584,0.07011733055114747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,3584,0.07741440137227376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,3072,0.06044906775156657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,3072,0.06240640083948771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,3072,0.06407466729482016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,2560,0.051159465312957765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,2560,0.05409066677093506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,65536,1.4723850250244142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,12288,0.2522069295247396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,16384,0.34479465484619143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,2048,0.041330134868621825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,51200,1.0850507100423177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,2048,0.04555413325627645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,1536,0.03962773482004801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,1536,0.042483198642730716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,1024,0.029157332579294842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,1024,0.03328640063603719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,6144,0.13077759742736816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,768,0.01941546599070231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,5120,0.10806079705556233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,768,0.027348266045252485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,512,0.015122133493423461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,512,0.024614399671554564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,4096,0.09008533159891764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,256,0.010443733135859171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,256,0.02123840053876241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,1536,0.03789013226826986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,128,0.008146133522192638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,128,0.019860267639160156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,128,0.013182933131853739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,64,0.007272533575693766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,64,0.02294293244679769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,4096,32,0.010328533252080281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,4096,32,0.022460800409317017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,2560,0.057290665308634435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,2048,0.04609813292821248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,65536,1.1416895548502604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,1024,0.029007999102274577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,65536,0.9643818537394205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,51200,0.851690673828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,51200,0.8167360305786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,16384,0.2629610697428385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,16384,0.2758314768473307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,768,0.02297813296318054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,12288,0.20541119575500488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,512,0.018232532342274985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,4096,256,0.01553813318411509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,10240,0.1651242733001709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,10240,0.16005226771036785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,8192,0.1372767925262451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,8192,0.12789759635925294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,7168,0.11873066425323486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,7168,0.11291413307189942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,12288,0.198853333791097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,6144,0.10146666367848714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,6144,0.09950400193532308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,12288,0.24193280537923179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,5120,0.08616853555043538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,5120,0.08581013679504394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,65536,1.3787456512451173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,10240,0.20348159472147623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,16384,0.3201792081197103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,4096,0.07388052940368653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,4096,0.07295893033345541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,3584,0.06549226840337118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,6144,0.12088533242543538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,3584,0.06527359882990519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,3072,0.05293866793314615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,3072,0.057912532488505045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,2560,0.044574932257334395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,2560,0.04973973433176677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,7168,0.14229119618733724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,2048,0.03679253260294597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,2048,0.042218665281931564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,51200,1.0614805221557617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,1536,0.03429439862569173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,1536,0.039877335230509445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,8192,0.164248530069987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,1024,0.02094506621360779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,1024,0.029070933659871418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,3072,0.06120213270187378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,768,0.017364267508188883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,768,0.026207999388376875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,1536,0.0321727991104126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,512,0.013481600085894265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,512,0.023387734095255533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,4096,0.08337706724802653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,256,0.009341866771380106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,2560,0.05334399938583374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,256,0.021336533625920615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,128,0.007653333246707916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,5120,0.10284053484598796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,128,0.01966080069541931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,64,0.007381333410739899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,64,0.020134399334589638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3584,32,0.0076906666159629825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3584,32,0.020308266083399452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,3584,0.07237760225931802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,1024,0.026365866263707478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,65536,0.9531690597534179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,2048,0.041971198717753094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,65536,0.9022997538248697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,65536,1.1269418080647786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,51200,0.7914890925089518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,51200,0.6973610560099284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,16384,0.22884693145751953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,12288,0.17351573308308918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,16384,0.25935680071512857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,51200,0.8524778366088868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,12288,0.18708799680074056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,10240,0.14185813268025715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,12288,0.2099221388498942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,10240,0.15624213218688965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,10240,0.1757962703704834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,8192,0.12382720311482746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,8192,0.12376000086466472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,7168,0.10480319658915202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,7168,0.10718506971995037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,128,0.012524799505869547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,256,0.014294399817784628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,6144,0.09171093304951985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,6144,0.09079466660817465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,6144,0.10769706567128498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,5120,0.07269759972890219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,5120,0.08146773179372152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,512,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,16384,0.2751925468444824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,4096,0.06785173416137695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,4096,0.07351786295572917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,3584,0.05315200090408325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,3584,0.06038719813028971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,3584,0.06284373203913371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3584,768,0.021285333236058555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,2560,0.038279465834299725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,3072,0.04604373375574748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,3072,0.05291733344395956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,2048,0.03118720054626465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,2560,0.044377601146698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,2048,0.03773546616236369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,1536,0.024012800057729086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,1536,0.03177066644032796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,8192,0.14792639414469402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,1536,0.02847359975179036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,1024,0.017296000321706136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,1024,0.02598506609598796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,1024,0.023177599906921385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,768,0.014111999670664468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,768,0.022924800713857017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,768,0.018780799706776936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,512,0.010484266281127929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,512,0.021541333198547362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,7168,0.1289621353149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,256,0.0074527998765309645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,4096,0.06002560059229532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,256,0.012903466820716858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,256,0.019644800821940103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,128,0.0065301333864529925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,128,0.018926932414372762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,64,0.006088533500830332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,128,0.01115733285744985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,64,0.01891626715660095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,3072,32,0.006415999929110209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,32,0.01864746610323588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,3072,0.05380906661351522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,3072,2048,0.038341331481933597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,65536,0.7467050552368164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,65536,0.8368736267089844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,51200,0.5774815877278645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,16384,0.2113877296447754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,16384,0.2142186641693115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,51200,0.6439178466796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,12288,0.14160854021708172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,5120,0.0865109364191691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,12288,0.16651946703592937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,16384,0.26978985468546546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,10240,0.11745920181274414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,10240,0.14061120351155598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,8192,0.09495039780934653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,8192,0.11373653411865234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,7168,0.08611946900685628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,7168,0.10046826998392741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,2560,0.04663360118865967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,6144,0.0734602689743042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,6144,0.08570559819539389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,3072,512,0.015161599715550741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,5120,0.06045973300933838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,5120,0.07414506276448568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,5120,0.08226133187611898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,4096,0.04934399922688802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,4096,0.061918934186299644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,65536,0.9991263707478841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,3584,0.04740266799926758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,3584,0.059368534882863366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,12288,0.19263572692871095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,51200,0.7968202590942383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,10240,0.15753386815388998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,3072,0.03856639862060547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,8192,0.12849600315093995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,2560,0.032790400584538776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,3072,0.05095786650975546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,2560,0.04211733341217041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,2048,0.02921280066172282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,2048,0.036586666107177736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,1536,0.021916800737380983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,1536,0.03131733338038127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,7168,0.11501546700795491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,1024,0.016572800278663636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,1024,0.02645866672197978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,6144,0.09868373076121012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,4096,0.06715520222981772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,768,0.013671466708183288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,768,0.023465599616368613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,3584,0.05931093295415243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,512,0.010712533195813497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,512,0.02218559980392456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,3072,0.04965013265609741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,256,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,2560,0.043380268414815265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,256,0.019552000363667808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,128,0.006626133124033611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,2048,0.03490666548411052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,128,0.0182261327902476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,64,0.005838933090368906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,64,0.018595200777053834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,1536,0.026317866643269856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2560,32,0.0061471998691558834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2560,32,0.018088533480962118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,1024,0.02100693384806315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,65536,0.5961727778116862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,65536,0.7291050593058268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,51200,0.46640958786010744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,16384,0.15338346163431804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,51200,0.6010677337646484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,51200,0.46694507598876955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,16384,0.2040170669555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,768,0.01723626653353373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,12288,0.11312747001647949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,16384,0.16454933484395345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,12288,0.1558293342590332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,10240,0.09503359794616699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,12288,0.11668586730957031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,10240,0.1296330690383911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,8192,0.07660799821217855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,10240,0.10166186491648357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,8192,0.10168853600819905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,8192,0.0828384002049764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,7168,0.06807786623636881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,7168,0.09174079895019531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,6144,0.05925013224283854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,6144,0.08110826810201009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,5120,0.049983998139699296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,5120,0.06838293075561523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,5120,0.05416320164998373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,4096,0.040166401863098146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,4096,0.05550719896952311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,4096,0.04577813148498535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,3584,0.03586879968643188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,3584,0.04899413188298543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,3072,0.030963200330734252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,3072,0.04319786628087362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,512,0.014354133605957031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,2560,0.026307199398676557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,2560,0.037523198127746585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,256,0.011788800358772278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,2560,0.031318400303522745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,2048,0.022210133075714112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,2048,0.032927999893824264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,1536,0.017819732427597046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2560,128,0.01043839951356252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,1536,0.022244266668955483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,1536,0.02849493424097697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,1024,0.013236266374588013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,1024,0.023798400163650514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,1024,0.01808639963467916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,768,0.011050666371981304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,768,0.02240426739056905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,768,0.015821866194407144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,512,0.008959999680519104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,512,0.02046826680501302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,512,0.01362986663977305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,256,0.0067007998625437425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,256,0.011434666315714518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,256,0.01837653319040934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,128,0.005909333129723867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,128,0.017195733388264973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,128,0.010493866602579753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,64,0.005186133086681366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,2048,32,0.005493333439032236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,64,0.017361066738764443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,2048,32,0.01727573275566101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,65536,0.4905077298482259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,65536,0.667254384358724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,65536,0.5968074798583984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,7168,0.07157866954803467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,6144,0.06290026505788168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,16384,0.12362666924794515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,51200,0.541155179341634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,16384,0.17642879486083984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,3584,0.04077226718266805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,12288,0.0927178700764974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,12288,0.13473386764526368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,3072,0.035556264718373615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,10240,0.07708160082499185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,10240,0.11559999783833821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,8192,0.06228586832682291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,2048,0.026948267221450807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,8192,0.09411626656850179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,51200,0.3733738581339518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,7168,0.05687786738077799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,7168,0.08315733273824057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,51200,0.4292437235514323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,6144,0.050578133265177405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,16384,0.1437983989715576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,6144,0.07108586629231771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,5120,0.042233598232269284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,12288,0.10796373685201008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,5120,0.060190931955973304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,4096,0.034295467535654704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,10240,0.09072533448537191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,4096,0.04932373364766439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,3584,0.030330665906270343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,3584,0.043280001481374106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,2048,65536,0.5962666829427083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,8192,0.07524480024973551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,3072,0.026474666595458985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,3072,0.038809601465861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,7168,0.06515626509984335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,2560,0.02249600092569987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,2560,0.034227200349171955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,2048,0.018279467026392618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,2560,0.028008532524108887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,1536,0.015007999539375306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,2048,0.024072533845901488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,1536,0.025329067309697466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,1024,0.010841600100199382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,1024,0.022363734245300294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,6144,0.05737386544545492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,768,0.009322667121887207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,768,0.02105600039164225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,2048,0.030825599034627275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,512,0.007505066692829132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,512,0.019927465915679933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,5120,0.04964160124460856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,256,0.005932799975077311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,256,0.018003199497858682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,4096,0.040299733479817704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,128,0.005017599960168203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,128,0.016849066813786825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,3584,0.03662506739298503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,128,0.00860479970773061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,64,0.004610133171081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1536,32,0.0048213332891464235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,64,0.017182934284210204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,3072,0.032052266597747806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1536,32,0.017307732502619425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,65536,0.3241045316060384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,1024,0.015987199544906617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,51200,0.25884906450907386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,768,0.013390933473904928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,51200,0.3346026738484701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,51200,0.4670453389485677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,16384,0.08101867039998373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,12288,0.061427199840545656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,16384,0.11094079812367756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,16384,0.160097074508667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,10240,0.052484265963236486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,12288,0.12140800158182781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,10240,0.07123733361562093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,8192,0.04198400179545085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,8192,0.08411093552907309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,1536,0.0188810666402181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,7168,0.03747626543045044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,512,0.01183573305606842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,7168,0.04897600015004476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,7168,0.07447360356648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,65536,0.5969888051350911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,6144,0.03216213385264079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,6144,0.06355520089467367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,5120,0.02728533347447713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,5120,0.052834133307139076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,5120,0.03667946656545003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,4096,0.02216426730155945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,4096,0.031678932905197146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,4096,0.0422815998395284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,3584,0.01987839937210083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,3584,0.037804798285166426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,3584,0.028182399272918702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,3072,0.01748266617457072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,3072,0.03434986670811971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,3072,0.024676267306009927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,2560,0.015313067038853965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,2560,0.030134399731953938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,2560,0.022235733270645142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,2048,0.012938666343688964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1536,256,0.009769599636395771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,2048,0.026825600862503053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,2048,0.018683733542760213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,10240,0.10633280277252197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,1536,0.010493866602579753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,1024,0.008380800485610962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,1024,0.013409066200256347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,1024,0.021512534221013388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,768,0.007187200089295705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,768,0.019633066654205323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,512,0.006082133452097575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,512,0.018361600240071614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,512,0.01014400025208791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,256,0.004923733572165171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,65536,0.41298774083455403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,256,0.017411200205485027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,1536,0.02401919960975647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,12288,0.08557759920756022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,128,0.0044938668608665465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,128,0.007747200131416321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,64,0.0040618665516376495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,128,0.016292267044385276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,1024,32,0.004265599946180979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,64,0.016612266500790916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,1024,32,0.01624853312969208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,65536,0.24795200030008951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,8192,0.05675839980443319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,6144,0.04447466532389323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,51200,0.19405867258707682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,65536,0.5652032216389974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,16384,0.0648693323135376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,16384,0.1498645305633545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,51200,0.4489738782246907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,16384,0.10068053404490154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,12288,0.0503114660580953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,12288,0.1158794641494751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,12288,0.07695146401723227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,1536,0.015441067020098367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,10240,0.041598931948343916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,8192,0.03385173479715983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,10240,0.09991679986317953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,8192,0.08041173617045085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,768,0.011879466970761617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,7168,0.030007465680440264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,7168,0.07104000250498453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,6144,0.026143999894460042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,6144,0.06006186803181966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,1024,256,0.008477866649627686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,5120,0.021741867065429688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,5120,0.04909546772638957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,4096,0.01841920018196106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,4096,0.03962666591008504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,65536,0.3787189483642578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,3584,0.016481066743532814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,3584,0.036055465539296463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,3584,0.025896533330281572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,51200,0.29730240503946936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,3072,0.014769066373507181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,3072,0.03214293320973714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,2560,0.01275200049082438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,2560,0.02895786762237549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,10240,0.0630741318066915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,2048,0.010870400071144103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,2048,0.025810132424036663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,2048,0.017025067408879598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,8192,0.05247360070546468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,1536,0.008899199962615966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,1536,0.02323413292566935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,1536,0.013916800419489542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,7168,0.047047468026479085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,1024,0.00727893312772115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,768,0.006397866706053417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,1024,0.020937599738438926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,768,0.010193066795667012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,768,0.020172800620396933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,512,0.018883200486501057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,6144,0.04128746589024861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,256,0.004911999901135763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,256,0.016875733931859337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,5120,0.034976001580556235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,128,0.004183466732501984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,4096,0.029240532716115313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,128,0.016124799847602844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,64,0.003702399879693985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,3072,0.023150932788848878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,64,0.016089600324630738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,32,0.00378560001651446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,768,32,0.01591146687666575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,65536,0.17076692581176758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,65536,0.5242762565612793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,768,512,0.005464533468087515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,51200,0.13664533297220866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,51200,0.4124757448832194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,16384,0.04915306568145752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,16384,0.14091839790344238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,2560,0.02040106654167175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,12288,0.034977066516876223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,12288,0.109879469871521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,1024,0.012276267011960346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,10240,0.03013226588567098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,512,0.009141332904497783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,10240,0.0929802656173706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,8192,0.024321067333221435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,256,0.00758186678091685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,8192,0.07501333554585775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,7168,0.021631999810536703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,768,128,0.006774400174617767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,7168,0.06439679861068726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,6144,0.018294399976730345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,6144,0.053717335065205894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,65536,0.36488320032755533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,5120,0.01520853340625763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,5120,0.044199466705322266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,51200,0.2851050694783529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,16384,0.09481066862742106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,4096,0.01283093293507894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,4096,0.03577920198440552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,12288,0.07362666924794516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,3584,0.01157866617043813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,3584,0.0321941335995992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,10240,0.062257067362467444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,3072,0.010708266496658325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,8192,0.05086506605148315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,3072,0.028483200073242187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,7168,0.04531626701354981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,2560,0.009435733159383137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,2560,0.02696640094121297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,2048,0.007867733140786488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,2048,0.024817067384719848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,6144,0.03987840016682943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,1536,0.007047466437021891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,1536,0.02193280061086019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,5120,0.03316799998283386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,1024,0.005824000140031179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,1024,0.019230933984120686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,768,0.005099733173847198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,768,0.018951465686162315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,4096,0.02736213405927022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,512,0.0044938668608665465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,512,0.01802133321762085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,3584,0.02528853416442871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,256,0.003868799904982249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,256,0.016301866372426352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,3072,0.02173653244972229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,128,0.003824000060558319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,2560,0.019547732671101888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,128,0.015572266777356467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,64,0.0034624000390370687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,2048,0.016408532857894897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,64,0.015931733449300132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,512,32,0.0035797332723935447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,512,32,0.01602453291416168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,65536,0.10766506989796956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,1536,0.013985066612561544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,51200,0.08642346858978271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,65536,0.4927797317504883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,51200,0.39120105107625325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,1024,0.011337600151697795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,16384,0.03365120093027751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,16384,0.13631893793741862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,12288,0.027821866671244304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,12288,0.10518613656361897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,768,0.009682133793830872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,10240,0.022805333137512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,10240,0.08908267021179199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,512,0.00848533312479655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,8192,0.018999467293421425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,8192,0.07034986813863119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,256,0.007291733225186665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,7168,0.016889599959055583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,7168,0.06036800146102905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,512,128,0.006790400048096975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,6144,0.015399466951688132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,6144,0.049763198693593344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,65536,0.32880961100260414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,5120,0.012269866466522217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,5120,0.0404586672782898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,51200,0.260425599416097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,4096,0.011201066772143047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,16384,0.0871071974436442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,4096,0.034578132629394534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,3584,0.01004266639550527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,3584,0.030322132507960002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,12288,0.06690346399943034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,3072,0.009351467092831928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,3072,0.028281599283218384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,10240,0.05713280042012533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,2560,0.008044800162315369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,2560,0.02651519974072774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,8192,0.04522346655527751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,2048,0.007260799904664357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,2048,0.023536000649134317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,7168,0.04146453142166138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,1536,0.006198399762312571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,1536,0.020987733205159505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,1536,0.012173866232236225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,1024,0.005258666475613912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,1024,0.01880853374799093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,6144,0.03625173171361287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,768,0.0046965335806210835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,768,0.018029866615931193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,768,0.008714666962623597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,512,0.004206933577855428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,4096,0.024755199750264488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,512,0.017010132471720375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,256,0.0035989334185918174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,256,0.015914666652679443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,3072,0.019631999731063842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,128,0.0033173332611719764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,128,0.015377066532770791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,2560,0.01750613252321879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,64,0.0031295999884605407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,2048,0.014677332838376364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,64,0.0151637335618337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,256,32,0.0032032000521818793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,256,32,0.01514240006605784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,65536,0.09610026677449544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,5120,0.030082132418950396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,65536,0.48418451944986984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,51200,0.07714347044626871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,1024,0.01015786627928416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,51200,0.3819648106892904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,16384,0.028521599372227986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,16384,0.13316266536712645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,3584,0.02228053410847982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,12288,0.023129600286483764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,12288,0.10342079798380535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,512,0.007670400043328603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,256,0.006680533289909363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,10240,0.016806399822235106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,10240,0.08710292975107828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,256,128,0.006027733286221823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,8192,0.0686474641164144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,7168,0.013739732901255288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,7168,0.057963732878367105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,7168,0.039803731441497806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,6144,0.012461866935094197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,6144,0.04782293240229289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,65536,0.3156693458557129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,5120,0.010859733819961548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,8192,0.015611732999483744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,5120,0.038894931475321456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,4096,0.009227733810742695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,4096,0.032391466697057086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,51200,0.24656747182210287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,3584,0.008891733487447102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,3584,0.02935360074043274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,3584,0.02039466698964437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,16384,0.07918612957000733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,3072,0.008162133395671844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,3072,0.027933865785598755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,2560,0.0070933332045873005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,2560,0.02495786746342977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,2560,0.016390400131543477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,2048,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,2048,0.0229418675104777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,12288,0.0641429344813029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,1536,0.005622399846712748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,1536,0.021178666750590006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,10240,0.054972799619038906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,1024,0.004671999812126159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,1024,0.01885546644528707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,8192,0.04377599954605103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,768,0.004282666742801667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,768,0.017666133244832356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,6144,0.033801599343617754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,512,0.0037087999284267426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,512,0.01684479912122091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,5120,0.02844053308169047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,256,0.0033514666060606635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,256,0.015958399573961893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,4096,0.02347946763038635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,128,0.0030986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,128,0.01532586713631948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,3072,0.018901334206263224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,64,0.0029663999875386557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,64,0.01518933375676473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,128,32,0.0030389333764712016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,128,32,0.014734933773676554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,65536,0.09277439912160237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,2048,0.014071466525395713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,51200,0.07387839953104655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,65536,0.4808234532674153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,1536,0.011967999736467998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,51200,0.3784768104553223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,12288,0.021316266059875487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,1024,0.009763200084368389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,16384,0.1305792013804118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,10240,0.014473600188891092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,12288,0.10066773096720379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,8192,0.01260693371295929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,10240,0.0844223976135254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,7168,0.011766399939854939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,6144,0.010973866780598958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,8192,0.06749119758605956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,7168,0.056686933835347494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,5120,0.009701333443323771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,6144,0.04764480193456014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,768,0.008496000369389852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,5120,0.03836160103480021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,4096,0.008594133456548055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,3584,0.007929599781831106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,3072,0.007162666817506154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,4096,0.03127359946568807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,2560,0.006427733103434245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,3584,0.029095466931660968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,2048,0.005625600119431814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,2560,0.024753065903981526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,3072,0.027352533737818402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,1536,0.005037866532802582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,1024,0.004249600072701773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,2048,0.022405334313710532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,1536,0.020756266514460244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,768,0.003882666677236557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,1024,0.01892906626065572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,512,0.003618133316437403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,768,0.01763199965159098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,256,0.0032799998919169106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,512,0.016787199179331462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,128,0.003020799905061722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,256,0.01623040040334066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,64,0.0028373333315054577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,128,0.01515733301639557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,32,0.0028607999285062153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,64,0.01478506624698639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,64,32,0.01499626636505127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,65536,0.09605866273244222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,51200,0.07482773462931315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,16384,0.026610134045283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,64,16384,0.027561599016189577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,16384,0.12981546719868978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,512,0.007285333176453908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,12288,0.020150399208068846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,10240,0.012972799936930337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,65536,0.48127040863037107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,8192,0.01144533356030782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,51200,0.3793994585673014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,12288,0.09973226388295492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,10240,0.08325333595275879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,7168,0.01032426655292511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,256,0.006279466549555461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,8192,0.06542293230692545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,5120,0.008706133564313252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,4096,0.0081194669008255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,6144,0.04512106577555339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,5120,0.03757973512013753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,3584,0.006909866631031036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,4096,0.03190720081329346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,3072,0.0065098668138186145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,3584,0.03001280029614766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,2560,0.00621973325808843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,3072,0.027506132920583088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,2560,0.024741333723068238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,1536,0.004935466746489207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,2048,0.02276479999224345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,1024,0.004154666761557261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,1536,0.020284799734751384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,768,0.003804799914360046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,1024,0.018441599607467652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,512,0.0035232000052928926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,256,0.0031776001056035364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,768,0.017322667439778647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,256,0.015597866972287497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,128,0.015177599589029946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,64,0.002889599899450938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,64,0.014829867084821067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,32,0.0028351999819278715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,32,0.01474453310171763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,6144,0.009372799595197042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,7168,0.05561173359553019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1793,128,128,0.005784533421198527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,2048,0.005434666574001312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1793,32,512,0.016502400239308677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1793,32,128,0.0030080000559488933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,16384,2.6405845642089845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,12288,2.057463455200195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,16384,3.4881568908691407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,12288,4.115499623616537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,16384,5.207880655924479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,12288,2.622605895996094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,10240,1.8280554453531903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,10240,3.368947092692057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,10240,2.2159467061360676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,8192,1.3192128499348958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,8192,2.5904490152994795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,8192,1.7377588907877606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,7168,1.262893803914388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,7168,2.2862388610839846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,7168,1.542678451538086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,51200,8.631109619140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,6144,1.075098673502604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,6144,1.334656016031901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,6144,1.9417524973551434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,5120,0.9248383839925131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,5120,1.0202229181925455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,5120,1.6658709208170575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,4096,1.3801141103108725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,4096,0.7846762975056966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,4096,0.862179183959961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,3584,0.6154848098754883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,3584,1.2061237335205077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,3072,0.5893386840820313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,3584,0.7616095860799154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,3072,0.6661631902058919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,3072,1.0053578694661458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,51200,11.796446736653646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,2560,0.4677568117777507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,2048,0.4130624135335286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,2560,0.6202943801879883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,2048,0.6704938888549805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,2560,0.9978645324707032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,1536,0.34079465866088865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,1024,0.3669621467590332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,1024,0.2484842618306478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,1536,0.46076478958129885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,1024,0.43118826548258465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,768,0.3032309214274088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,768,0.2103893280029297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,768,0.34164158503214515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,512,0.17727573712666828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,512,0.24119253158569337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,512,0.2504415988922119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,256,0.14991359710693358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,256,0.12521173159281412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,128,0.10373653570810955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,256,0.23431466420491537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,128,0.11160000165303548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,64,0.09628159999847412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,32,0.09564586480458578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,128,0.20898987452189127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,32,0.10783147017161052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,1536,0.5384682973225912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,65536,64,0.11017386913299561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,65536,51200,18.348640950520835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,16384,4.0373598734537754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,51200,6.707833353678386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,65536,2048,0.45311679840087893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,65536,8.855111694335937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,16384,2.3032457987467447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,51200,8.471972147623699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,12288,1.7066421508789062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,12288,3.059434763590495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,12288,2.083798344930013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,10240,1.6594165802001952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,51200,12.836915079752604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,10240,2.6176180521647137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,10240,1.6548800150553387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,65536,13.619139607747396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,8192,2.355852762858073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,8192,1.2182112375895182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,7168,0.9495370864868165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,7168,1.2053034464518229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,8192,1.4212384541829428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,6144,0.8156554539998373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,6144,0.97979736328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,7168,2.216033172607422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,6144,1.7464810689290364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,5120,0.7312938690185546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,5120,1.3682080586751302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,5120,0.8680778503417969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,4096,0.5743647893269856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,65536,17.43019002278646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,4096,1.013651212056478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,4096,0.7402058919270833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,3584,0.9216874440511068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,3584,0.5488138834635417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,3584,0.6453962961832682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,3072,0.8387104034423828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,3072,0.5124352137247722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,3072,0.42977174123128253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,2560,0.40171626408894856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,2560,0.6701194763183593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,2560,0.502513058980306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,2048,0.3454005241394043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,2048,0.5230293273925781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,2048,0.3877333323160807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,1536,0.40256427129109695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,1536,0.265012264251709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,1536,0.3207093238830566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,1024,0.28748906453450523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,1024,0.19523626963297527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,1024,0.2723957379659017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,768,0.21619946161905923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,512,0.1557792027791341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,768,0.1633301258087158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,768,0.21677120526631674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,512,0.1320522705713908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,256,0.11402346293131511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,512,0.18208959897359211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,256,0.0992415984471639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,128,0.08331519762674967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,64,0.07738239765167236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,128,0.09109333356221518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,128,0.14249599774678548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,51200,32,0.0770581324895223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,64,0.09080959955851237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,51200,32,0.0890613317489624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,256,0.14562880198160807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,51200,16384,2.7039723714192707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,51200,2.4511423746744794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,16384,1.201041030883789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,51200,2.700152587890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,16384,0.7481386820475261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,65536,3.016593170166016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,12288,0.6203989028930664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,16384,0.8472426732381185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,12288,0.6139370600382488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,65536,3.643818664550781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,10240,0.8289386749267578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,10240,0.4771925290425618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,10240,0.5283040046691895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,8192,0.4129621187845866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,8192,0.6375680287679036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,51200,4.7077484130859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,8192,0.452286942799886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,7168,0.5478442509969075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,7168,0.38811734517415364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,7168,0.37976214090983074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,6144,0.4829610824584961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,6144,0.30473171869913734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,6144,0.33453760147094724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,12288,0.9926634470621745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,5120,0.26611092885335286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,5120,0.4121098518371582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,5120,0.2987903912862142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,4096,0.3122922579447428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,4096,0.22797867457071938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,3584,0.28321812947591146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,3584,0.20495146115620932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,4096,0.21692159970601402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,3584,0.22616426150004068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,3072,0.1706293265024821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,3072,0.2548746744791667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,2560,0.13639039993286134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,3072,0.1945674737294515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,2048,0.1627690633138021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,2048,0.12919999758402506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,2048,0.12319680054982503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,1536,0.13519466718037923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,1024,0.08730453650156657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,1536,0.12762666543324788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,1024,0.08134826819101969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,1024,0.08137813409169516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,768,0.08095040321350097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,768,0.06612159808476767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,512,0.0605130672454834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,768,0.0723146677017212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,512,0.054080001513163244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,512,0.07341226736704508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,256,0.04521600008010864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,256,0.061543468634287515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,128,0.030034132798512775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,65536,5.21907958984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,128,0.042854400475819905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,128,0.05044480164845785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,32,0.02842666705449422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,64,0.042181332906087235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,64,0.030298666159311934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,32,0.04251840114593506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,2560,0.20576213200887045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,16384,1536,0.09719893137613932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,16384,256,0.0345909317334493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,16384,2560,0.16646933555603027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,16384,0.993446413675944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,16384,0.5949301401774089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,65536,2.528675079345703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,51200,1.9007029215494793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,12288,0.7110080083211263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,12288,0.4584469477335612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,51200,3.2179275512695313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,10240,0.38818133672078453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,10240,0.6578058878580729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,65536,4.074561055501302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,8192,0.48502613703409836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,8192,0.3473151842753092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,7168,0.29238828023274743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,7168,0.4361685434977214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,6144,0.3876949310302734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,6144,0.26820907592773435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,12288,0.5530346552530925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,5120,0.2929312070210775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,5120,0.20335146586100258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,16384,0.7170698801676433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,4096,0.2595978736877441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,4096,0.1788266658782959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,65536,3.2866111755371095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,3584,0.2340714613596598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,10240,0.49318933486938477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,6144,0.28032000859578454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,3072,0.18083945910135907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,3072,0.1272117296854655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,51200,2.41691411336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,2560,0.16565653483072917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,7168,0.34468053181966146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,2560,0.12469546794891358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,2048,0.13242560227711994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,2048,0.09859840075174967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,3584,0.1625216007232666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,1536,0.09701013565063477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,8192,0.3714058558146159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,1536,0.07826773325602213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,1024,0.06831466356913249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,1024,0.060660266876220705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,3072,0.14064426422119142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,768,0.0530847986539205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,768,0.05151786804199219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,4096,0.19523199399312335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,512,0.039485867818196616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,512,0.043798398971557614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,1536,0.0775274674097697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,256,0.02712106704711914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,256,0.03760639826456706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,5120,0.23137493133544923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,128,0.02347093423207601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,2560,0.12617599964141846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,128,0.03477226495742798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,3584,0.16651733716328937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,64,0.021523199478785195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,64,0.03529173135757446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,12288,32,0.022823466857274374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,12288,32,0.0351583997408549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,1024,0.0636789321899414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,2048,0.1031989336013794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,65536,2.253271484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,65536,2.2947263081868488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,65536,3.439863586425781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,256,0.03464959859848023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,128,0.029257599512736005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,16384,0.8329834620157877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,16384,0.5183103879292805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,51200,1.7602101643880208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,51200,2.78353271484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,512,0.04032426675160726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,16384,0.5214399973551432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,12288,0.40589440663655596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,51200,1.713861338297526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,10240,0.3316181182861328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,12288,0.43191572825113933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,10240,0.3820320129394531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,10240,0.5472298940022786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,8192,0.4011562665303548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,8192,0.2724085489908854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,7168,0.39099200566609704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,7168,0.2766357421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,7168,0.25010560353597006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,6144,0.2181429386138916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,6144,0.32191893259684246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,12288,768,0.05024106502532959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,5120,0.18512746493021648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,5120,0.18473706245422364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,5120,0.27837438583374025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,4096,0.22630292574564614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,4096,0.14836160341898602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,4096,0.14295040766398112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,3584,0.13647573788960773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,3584,0.18502933184305828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,3584,0.1504543940226237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,3072,0.15498132705688478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,3072,0.11266880035400391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,3072,0.11919573148091633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,2560,0.13617706298828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,2560,0.1105013370513916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,2560,0.1005845308303833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,2048,0.10524053573608398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,1536,0.09037333329518636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,2048,0.0938431978225708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,12288,0.626699701944987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,1536,0.07157653172810872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,1536,0.06789013544718424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,1024,0.060494931538899746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,1024,0.06082026561101278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,768,0.04633920192718506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,768,0.04701226552327474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,512,0.03856000105539958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,1024,0.05424960056940714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,512,0.04095466534296672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,256,0.02568853298823039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,512,0.04065066576004028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,256,0.03810453414916992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,128,0.024218666553497314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,256,0.03533226648966471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,8192,0.31063572565714515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,64,0.02269866665204366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,128,0.0321397324403127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,128,0.035043199857076004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,10240,32,0.01832533280054728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,64,0.03173013329505921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,6144,0.2313279946645101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,2048,0.08579520384470621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,10240,768,0.05266346534093221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,10240,32,0.03177599906921387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,65536,1.6862528483072918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,16384,0.5915840148925782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,51200,1.3422730763753257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,65536,1.8277109781901042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,16384,0.42025279998779297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,51200,2.095083745320638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,12288,0.3577930768330892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,12288,0.4820608139038086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,10240,0.3742741266886393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,16384,0.4411050796508789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,51200,1.4087103525797526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,12288,0.36499201456705727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,10240,0.30388161341349285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,65536,2.7058497111002606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,8192,0.21617600123087563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,10240,0.285368537902832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,8192,0.32396373748779295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,8192,0.22636799812316893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,7168,0.21745600700378417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,7168,0.3063839912414551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,6144,0.17395520210266113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,6144,0.17863465944925944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,6144,0.2412639935811361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,5120,0.2083903948465983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,5120,0.14398825963338216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,5120,0.15347946484883626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,4096,0.1720138708750407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,4096,0.12206933498382569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,4096,0.1277013301849365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,3584,0.1487829367319743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,3584,0.10484480063120524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,3584,0.1089898665746053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,3072,0.1300928036371867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,3072,0.09250240325927735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,3072,0.09723626772562663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,2560,0.1072544018427531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,2048,0.08572373390197754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,2560,0.08015039761861166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,2048,0.06837013562520346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,1536,0.06720000108083089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,2048,0.07416533629099528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,1024,0.042650667826334636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,1536,0.056618666648864745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,1024,0.04691946506500244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,1024,0.04692693154017131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,768,0.03389973243077596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,768,0.03809386491775513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,512,0.025916800896326704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,768,0.04566826820373535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,512,0.032927999893824264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,256,0.018040533860524496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,512,0.033986131350199386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,128,0.014613333344459533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,256,0.03020693262418111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,256,0.0287882665793101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,64,0.013809067010879517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,128,0.027292799949645997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,8192,32,0.015046399831771851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,64,0.028600533803304035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,32,0.02780906756718953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,7168,0.20539093017578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,2560,0.0838805357615153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,8192,1536,0.06091306606928507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,8192,128,0.026791467269261675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,65536,1.5069557189941407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,65536,1.595705540974935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,16384,0.5251210530598958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,51200,1.7468437194824218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,16384,0.394926929473877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,16384,0.4069685300191243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,51200,1.167969004313151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,51200,1.1651925404866537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,12288,0.4144693374633789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,12288,0.28564907709757487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,12288,0.31064319610595703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,8192,0.26213119824727377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,10240,0.2598752021789551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,10240,0.2785557428995768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,8192,0.19800747235616048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,8192,0.23631253242492675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,7168,0.17606933911641437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,7168,0.17528640429178874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,6144,0.23457066218058267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,6144,0.15675412813822429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,5120,0.16725759506225585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,5120,0.13434774080912273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,5120,0.13187519709269208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,6144,0.1622389316558838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,4096,0.13574612935384114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,4096,0.10926933288574218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,4096,0.10365973313649494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,3584,0.12469546794891358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,3584,0.10752426783243815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,3072,0.10383040110270184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,3584,0.0938431978225708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,3072,0.08387733300526937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,3072,0.08876799742380778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,2560,0.07654933134714761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,2048,0.07134933471679687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,2560,0.07057920296986898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,2048,0.0670741319656372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,65536,2.337144470214844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,2048,0.060420266787211095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,1536,0.05547413428624472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,1536,0.05147093137105306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,1024,0.044241066773732504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,1024,0.04180266857147217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,1024,0.04071679910024007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,768,0.033766400814056394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,512,0.025389866034189863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,768,0.0363317330678304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,768,0.0384768009185791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,512,0.03380693197250366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,512,0.02935146689414978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,256,0.028899200757344562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,10240,0.35445547103881836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,128,0.014349866906801859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,64,0.012306132912635803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,128,0.025278933842976886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,128,0.023536000649134317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,32,0.012820266683896384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,64,0.025943466027577716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,7168,0.22748586336771645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,32,0.025934932629267375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,2560,0.08952319622039795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,7168,1536,0.05466986497243246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,7168,256,0.018312533696492515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,7168,256,0.02585279941558838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,65536,1.3684810638427733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,16384,0.45819412867228193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,16384,0.3630101203918457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,65536,2.0296245574951173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,51200,1.081933848063151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,51200,1.5786378224690756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,12288,0.34242773056030273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,12288,0.2961706797281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,10240,0.24732160568237305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,10240,0.33235413233439126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,8192,0.23519573211669922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,8192,0.21392532984415688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,7168,0.21364266077677407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,7168,0.15896426836649577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,6144,0.17370880444844564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,6144,0.13711786270141602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,12288,0.3095914522806803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,5120,0.14786027272542318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,5120,0.11942719618479411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,65536,1.6831114451090496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,16384,0.3795776049296061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,4096,0.1267850637435913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,4096,0.10035946369171142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,6144,0.14793920516967773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,3584,0.10649279753367107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,3584,0.08905173142751058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,3072,0.08943999608357747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,10240,0.2632383982340495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,3072,0.07881386280059814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,2560,0.076146133740743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,2560,0.06901973088582357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,7168,0.1732192039489746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,2048,0.06308906475702922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,2048,0.05975360075632731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,51200,1.2682132720947266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,1536,0.052894934018452966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,1536,0.05356160004933676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,8192,0.1941450595855713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,1024,0.03482666810353597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,1024,0.03877973159154256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,3072,0.07561386426289876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,768,0.027719465891520183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,768,0.034354134400685625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,1536,0.04375040133794149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,4096,0.10273706912994385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,512,0.020924800634384157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,512,0.03274346590042114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,256,0.016612266500790916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,256,0.027564799785614012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,2560,0.0681279977162679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,128,0.013913599650065103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,128,0.026124799251556398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,5120,0.12394773165384929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,64,0.012011733651161195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,64,0.026372265815734864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,6144,32,0.012290133039156596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,6144,32,0.02585279941558838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,3584,0.08924586772918701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,1024,0.037361065546671554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,2048,0.056279468536376956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,65536,1.2463136037190756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,65536,1.7345055898030597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,51200,1.264638900756836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,51200,0.9806432088216146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,16384,0.45497919718424484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,51200,1.1243935902913411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,16384,0.333898671468099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,16384,0.3791914621988932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,12288,0.32967039744059246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,12288,0.2502858638763428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,12288,0.2678784052530924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,10240,0.26590399742126464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,10240,0.20844693183898927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,8192,0.21490880648295083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,8192,0.1688810666402181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,7168,0.17609814008076985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,7168,0.15411413510640462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,128,0.01581546664237976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,256,0.01845653255780538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,6144,0.14818026224772135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,6144,0.12463146845499676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,6144,0.125382399559021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,5120,0.12637120087941486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,5120,0.10780800183614095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,4096,0.1015775998433431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,4096,0.09069759845733642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,65536,1.364467239379883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,3584,0.10248853365580242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,512,0.023548799753189086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,3584,0.09522240161895752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,6144,768,0.02810346682866414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,3072,0.08367466926574707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,3072,0.07377599875132243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,2560,0.06694933573404947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,2560,0.06485866705576579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,10240,0.21213226318359374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,8192,0.16618666648864747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,2048,0.05530346632003784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,2048,0.05599466562271118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,7168,0.14629653294881184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,1536,0.04353066682815552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,1536,0.046522665023803714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,5120,0.10536320209503174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,1024,0.031062400341033934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,1024,0.03660373290379842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,4096,0.08619200388590495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,768,0.025684267282485962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,768,0.03232000072797139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,768,0.024821333090464272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,3584,0.07590933640797934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,512,0.019760000705718993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,512,0.028539733091990156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,256,0.01393066644668579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,256,0.024617600440979003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,3072,0.06480853160222372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,256,0.016569599509239197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,128,0.010163199901580811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,128,0.02169706622759501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,64,0.008778666456540424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,64,0.02204266587893168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,5120,32,0.009124267101287841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,5120,32,0.021923200289408366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,2560,0.05925333499908447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,2048,0.04836053450902303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,65536,1.0905408223470052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,65536,1.3565109252929688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,1536,0.037564798196156816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,16384,0.3235093434651693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,51200,0.8640010833740235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,51200,1.0050474802652993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,16384,0.2853952089945475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,1024,0.03250453273455302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,12288,0.2615776062011719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,12288,0.21793386141459145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,16384,0.2696607907613119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,12288,0.2152224063873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,512,0.020357332626978555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,8192,0.16267093022664386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,10240,0.21478613217671713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,8192,0.14578666687011718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,7168,0.1520821412404378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,5120,128,0.01373546620210012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,7168,0.12833600044250487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,6144,0.12394560178120931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,6144,0.11158186594645184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,5120,0.10258560180664063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,5120,0.09644266764322916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,10240,0.183186133702596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,4096,0.08505813280741373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,4096,0.08160320123036703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,65536,1.1582528432210286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,3584,0.0921183983484904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,3584,0.08846506277720133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,51200,0.8817706425984702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,3072,0.07517440319061279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,3072,0.07418879667917887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,10240,0.1864533265431722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,2560,0.055181864897410074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,8192,0.13903679847717285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,2560,0.056245334943135585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,2048,0.04535359938939412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,7168,0.12227520147959392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,2048,0.04768426815668742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,6144,0.10520746707916259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,1536,0.035524264971415205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,1536,0.039162667592366536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,1024,0.025534933805465697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,1024,0.03210773269335429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,5120,0.08857066631317138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,768,0.02115946610768636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,768,0.028818132479985555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,4096,0.07343893051147461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,512,0.016661333044370015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,512,0.024617600440979003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,3584,0.06520746548970541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,256,0.011377066373825073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,256,0.02235306700070699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,256,0.013988266388575235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,128,0.008550399541854858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,128,0.02042133410771688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,3072,0.0550816019376119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,64,0.007258666555086772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,64,0.02068906625111898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,4096,32,0.007978666822115581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,4096,32,0.020606933037439983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,2560,0.049379201730092366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,2048,0.04070933262507121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,1536,0.031883732477823896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,1024,0.027137066920598345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,768,0.02102400064468384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,65536,1.149127451578776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,65536,0.8367178599039713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,65536,0.9833909352620444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,51200,0.915555191040039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,16384,0.3087648073832194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,51200,0.7685216267903645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,16384,0.2800874710083008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,12288,0.22776212692260742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,12288,0.19326400756835938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,12288,0.1790463924407959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,10240,0.18974612553914388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,512,0.01691626707712809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,10240,0.16563199361165365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,10240,0.14694933891296386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,8192,0.14712106386820475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,7168,0.1256767988204956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,7168,0.1291061321894328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,6144,0.10963306427001954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,6144,0.10754133065541585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,6144,0.08757440249125162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,5120,0.09193493525187174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,4096,128,0.011987200379371643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,5120,0.08979307015736898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,5120,0.07230719725290934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,4096,0.07363626956939698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,4096,0.07479573090871175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,4096,0.059699201583862306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,3584,0.06503680149714151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,3584,0.06702933311462403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,3072,0.05640000104904175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,3072,0.058974933624267575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,8192,0.13172799746195477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,3072,0.04810880025227864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,2560,0.04791253407796224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,2560,0.0512661337852478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,2560,0.0410805344581604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,2048,0.03887253204981486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,2048,0.03582826852798462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,2048,0.04257173140843709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,1536,0.0295413335164388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,1536,0.03624000151952107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,1536,0.0298250675201416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,1024,0.02177706758181254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,1024,0.030181332429250078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,1024,0.023819732666015624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,768,0.017479467391967773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,768,0.02744106650352478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,768,0.021452800432840983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,512,0.01384106675783793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,512,0.026190932591756182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,512,0.017499732971191406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,256,0.010337066650390626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,256,0.014987732966740927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,256,0.022683733701705934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,128,0.009406933188438415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,128,0.02109439969062805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,128,0.013979732990264893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,64,0.0086325337489446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,64,0.022046933571497597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3584,32,0.009306666254997254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3584,32,0.021511467297871907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,65536,0.9197269439697265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,65536,0.8985002517700196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,65536,0.9356320063273111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,51200,0.7796682357788086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,51200,0.7070794423421224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,51200,0.7645792007446289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,16384,0.25331840515136717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,16384,0.2380352020263672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,12288,0.18791252772013348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,12288,0.18032639821370441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,10240,0.14593173662821451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,10240,0.14924480120340983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,51200,0.6239722569783528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,8192,0.12684693336486816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,8192,0.13010773658752442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,8192,0.12387413183848064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,7168,0.10960319836934407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,7168,0.11597013473510742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,16384,0.2282709280649821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,6144,0.08884373505910238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,12288,0.1731669267018636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,6144,0.09302079677581787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,7168,0.09829013347625733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,5120,0.0741973320643107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,5120,0.08017066319783529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,4096,0.06067306598027548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,4096,0.06768213113149008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,10240,0.153056001663208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,16384,0.2094559987386068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,3584,0.05345493157704671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,3584,0.05973759889602661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,3584,0.054635731379191074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,3072,0.04546026786168416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,3072,0.05297706524531046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,2560,0.038726401329040525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,2560,0.04495466550191243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,8192,0.11007253328959148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,2048,0.03120853304862976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,2048,0.03806826670964559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,2048,0.03297599951426188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,1536,0.024336000283559166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,1536,0.032076799869537355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3584,3584,0.05418560107549032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,1024,0.01804693341255188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,7168,0.10817920366923015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,1024,0.02871146599451701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,6144,0.09312427043914795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,768,0.014457600315411887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,5120,0.07478400071461996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,768,0.025569067398707075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,512,0.010640000303586323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,512,0.023093332846959434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,4096,0.06285866498947143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,256,0.00881599982579549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,256,0.020399999618530274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,3072,0.04534080028533936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,128,0.007303466896216075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,128,0.018832000096638997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,2560,0.04154986540476481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,128,0.010145066181818645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,64,0.006133333345254262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,3072,32,0.006478933493296306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,64,0.018918399016062418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,3072,32,0.01921280026435852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,65536,0.7871146519978841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,65536,0.8426666895548502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,1536,0.025868799289067584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,51200,0.6247498830159505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,1024,0.022100265820821127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,51200,0.6566154479980468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,16384,0.20287893613179525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,16384,0.22201919555664062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,12288,0.14577813148498536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,12288,0.16875413258870442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,768,0.017378133535385133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,10240,0.1196010669072469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,10240,0.14286293983459472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,512,0.013931733369827271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,8192,0.09695893128712972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,8192,0.11487893263498943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,3072,256,0.011517866452534994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,7168,0.08655359745025634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,7168,0.1015658696492513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,6144,0.07458773454030355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,6144,0.0892416000366211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,12288,0.16546026865641278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,5120,0.06252906719843546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,5120,0.07402986685434977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,65536,0.9021823883056641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,16384,0.21937173207600913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,4096,0.050218665599823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,6144,0.08513173262278238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,4096,0.06115093231201172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,3584,0.04404693444569906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,3584,0.05396053393681845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,3072,0.037836798032124835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,3072,0.046911998589833574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,10240,0.14465813636779784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,2560,0.03188160061836243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,2560,0.04076693455378215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,7168,0.0967093308766683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,2048,0.026282666126887004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,2048,0.034842665990193686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,51200,0.6999776204427083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,1536,0.022272000710169472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,1536,0.03144213358561198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,8192,0.10809066295623779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,1024,0.0151936004559199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,1024,0.02494186758995056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,3072,0.04137493371963501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,768,0.012272000312805176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,768,0.022612265745798745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,1536,0.02431466579437256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,512,0.009380267063776652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,512,0.02062826752662659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,4096,0.05633813142776489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,256,0.006761600077152252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,256,0.01879040002822876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,2560,0.03663146495819092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,128,0.005832533538341522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,128,0.01889066696166992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,5120,0.06871679623921713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,64,0.005994666616121928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,3584,0.046561066309611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,64,0.019516799847284952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2560,32,0.006361599763234456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2560,32,0.018734933932622273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,1024,0.020232532421747842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,2048,0.03009919921557109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,65536,0.6275210698445638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,65536,0.7522272109985352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,51200,0.4961717287699382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,16384,0.1623125394185384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,51200,0.5600085576375325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,16384,0.20540053049723306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,12288,0.12300266424814861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,12288,0.15737600326538087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,12288,0.13080213069915772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,10240,0.10152746836344402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,10240,0.12776959737141927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,51200,0.5958719889322917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,8192,0.08169600168863932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,8192,0.10618133544921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,128,0.010097066561381023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,65536,0.7012821197509765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,256,0.011318399508794149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,7168,0.07181119918823242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,7168,0.09257493019104004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,6144,0.06169066826502482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,16384,0.1729002634684245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,6144,0.0802506685256958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,5120,0.05209706624348959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,512,0.013473066687583923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,5120,0.05856000185012818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,4096,0.04342293341954549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,4096,0.05789120197296142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,10240,0.11622506777445477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,3584,0.038210133711496994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,3584,0.05073279937108358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,8192,0.09456533590952555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,3072,0.03336853186289469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,3072,0.04503999948501587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,5120,0.06916800340016684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,2560,0.028065067529678345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,2560,0.037913600603739425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,7168,0.08319573402404785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,2048,0.02373973329861959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2560,768,0.016028799613316855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,2048,0.03410346508026123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,1536,0.01922453244527181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,6144,0.06911679903666178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,1536,0.02993920048077901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,1024,0.014472533265749613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,1024,0.017192532618840538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,1024,0.024499199787775674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,768,0.011941333611806233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,768,0.02127893368403117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,768,0.014387200276056925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,512,0.008266666531562805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,512,0.019749333461125694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,256,0.006138666470845541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,256,0.018228266636530557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,4096,0.049965866406758624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,128,0.005400533477465311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,128,0.01755946675936381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,3584,0.04198186794916789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,64,0.005130666494369507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,64,0.017573332786560057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,2048,32,0.005355733136336008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,2048,32,0.01739306648572286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,3072,0.03601386547088623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,65536,0.4573429425557454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,2560,0.03264960050582886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,51200,0.3544266700744629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,65536,0.6620533625284831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,51200,0.5305930773417156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,16384,0.11599786281585693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,2048,0.02725013295809428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,16384,0.17730453809102376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,1536,0.021320533752441407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,12288,0.0879103978474935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,12288,0.13370025952657064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,10240,0.07366186777750651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,512,0.011875200271606445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,10240,0.11371946334838867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,8192,0.0595082680384318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,8192,0.09213013648986816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,128,0.00897813340028127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,7168,0.053464531898498535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,7168,0.08363413016001384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,2048,256,0.010244266192118327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,6144,0.0456767996152242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,6144,0.07169600327809653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,12288,0.1218389352162679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,5120,0.03832426468531291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,65536,0.6406933466593425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,5120,0.06055466731389364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,4096,0.031093333164850873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,4096,0.048972801367441816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,16384,0.16350293159484863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,3584,0.027668267488479614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,3584,0.04328639904658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,10240,0.103603196144104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,3072,0.023942399024963378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,51200,0.5003754615783691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,3072,0.03863999843597412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,2560,0.02100800077120463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,2560,0.03400213321050008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,6144,0.05850559870402018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,2048,0.017772799730300902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,7168,0.06961920261383056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,2048,0.030504532655080158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,1536,0.014436266819636025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,1536,0.025380265712738038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,8192,0.07764480113983155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,1024,0.010572800040245056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,1024,0.02239039937655131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,3072,0.030772266785303752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,768,0.009085866808891296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,768,0.02100373307863871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,4096,0.041553068161010745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,512,0.007340799768765767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,512,0.01950826644897461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,1536,0.018285866578420004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,2560,0.027085866530736285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,256,0.0059125334024429325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,256,0.01772693395614624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,3584,0.035403732458750406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,128,0.005067733426888784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,128,0.016548267006874083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,1024,0.015082666277885437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,64,0.004546133180459341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,64,0.016950400670369466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,5120,0.04907413323720296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1536,32,0.004763733347256978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1536,32,0.017072000106175742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,2048,0.023179733753204347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,65536,0.31817706425984704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,51200,0.25967466036478676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,65536,0.5983936309814453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,51200,0.4953578631083171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,16384,0.0840714693069458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,16384,0.15760960578918456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,12288,0.06363093455632528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,12288,0.12357119719187419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,65536,0.4765002568562825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,10240,0.05430399974187215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,256,0.008975999553998311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,512,0.010733866691589355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,10240,0.10457706451416016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,8192,0.04277226527531942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,128,0.008105599880218506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,10240,0.07990399996439615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,7168,0.03806399901707967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,8192,0.08536213239034017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,51200,0.3695551872253418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,7168,0.0747434695561727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,6144,0.033701332410176595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,5120,0.028153600295384724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,6144,0.06486080090204874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,6144,0.050910933812459314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1536,768,0.012657066186269125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,5120,0.05321066776911417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,4096,0.022792534033457438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,5120,0.044250667095184326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,3584,0.020678400993347168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,4096,0.04303786754608154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,3584,0.03858240048090617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,3072,0.018563199043273925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,16384,0.12900266647338868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,2560,0.0162581334511439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,2560,0.031023999055226643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,2560,0.024435200293858848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,2048,0.01346560021241506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,12288,0.09810986518859863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,2048,0.02738879919052124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,1536,0.010939733187357584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,1536,0.024233599503835045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,3072,0.03549866676330567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,1024,0.008586666981379191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,1024,0.021079466740290324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,7168,0.05914346774419149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,768,0.007517866790294647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,8192,0.07011520067850749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,768,0.01987733244895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,768,0.010770133137702942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,512,0.006287999947865804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,512,0.01844053268432617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,256,0.004990933338801066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,3584,0.032253867387771605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,4096,0.03629013299942017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,256,0.017110399405161538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,128,0.004221866528193155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,128,0.015918933351834617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,3072,0.02837973237037659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,64,0.003991466760635376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,1024,32,0.004231466849644979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,64,0.016331733266512553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,1024,32,0.016385066509246825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,65536,0.23773226737976075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,2048,0.02059626579284668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,1536,0.016004266341527303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,51200,0.1883061408996582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,65536,0.5607413609822591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,16384,0.06398293177286783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,51200,0.44578027725219727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,16384,0.14779733022054037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,12288,0.04866880178451538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,12288,0.12580053011576336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,1024,0.013032533725102744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,10240,0.04150826533635457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,10240,0.1069216012954712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,512,0.009289600451787313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,8192,0.03341226577758789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,256,0.007782400151093801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,8192,0.058124800523122154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,8192,0.08632427056630453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,1024,128,0.006899199883143107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,7168,0.041206399599711105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,7168,0.07515412966410319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,6144,0.03648213148117065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,6144,0.04521493514378865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,6144,0.06413760185241699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,5120,0.03006826639175415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,5120,0.05208746592203776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,5120,0.038341331481933597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,4096,0.02500480016072591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,4096,0.04193386634190877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,65536,0.4184842745463054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,3584,0.021927465995152794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,51200,0.32784639994303383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,3584,0.037818666299184164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,3072,0.019661867618560792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,3584,0.027820799748102826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,3072,0.0341045339902242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,16384,0.10980587005615235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,2560,0.016429866353670754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,2560,0.02132693330446879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,2560,0.029796266555786134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,2048,0.013692800203959146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,2048,0.02669866681098938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,1536,0.011014399925867717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,1536,0.023425066471099855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,12288,0.08772799968719483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,1536,0.014392532904942832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,1024,0.007944533228874206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,1024,0.02069013317426046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,768,0.00698773314555486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,768,0.019259732961654664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,10240,0.0720309336980184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,512,0.005725866556167603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,512,0.018132267395655315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,512,0.00858026643594106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,256,0.004755199948946635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,256,0.017426133155822754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,7168,0.05154773394266764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,128,0.00432640016078949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,128,0.015757866700490317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,128,0.006405333181222279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,64,0.003964799890915553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,64,0.015915733575820924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,768,32,0.0042463997999827065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,768,32,0.015549866358439126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,4096,0.03174293239911397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,65536,0.17094292640686035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,51200,0.13588693936665852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,3072,0.024590933322906496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,65536,0.5219615936279297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,2048,0.018363734086354576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,16384,0.04993813435236613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,51200,0.4228874524434407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,16384,0.14320640563964843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,12288,0.03726293245951335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,12288,0.11107412974039714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,12288,0.06718826293945312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,10240,0.031658667325973514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,1024,0.011954133709271748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,8192,0.024675200382868447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,768,0.009911466638247173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,8192,0.07467733224232992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,7168,0.021335466702779134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,7168,0.06470506588617961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,10240,0.09433279832204183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,6144,0.01834026575088501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,6144,0.053955201307932535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,768,256,0.007079466680685679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,5120,0.01581013302008311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,5120,0.04463466803232829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,65536,0.32035627365112307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,4096,0.013451733191808066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,4096,0.036084266503651936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,4096,0.024551467100779215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,3584,0.012173866232236225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,51200,0.2569098631540934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,3584,0.03253013292948405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,16384,0.08496320247650146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,3072,0.029977599779764812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,3072,0.01995733380317688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,2560,0.009499733646710713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,2560,0.027872000137964887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,10240,0.0563594659169515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,2048,0.008297599852085114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,2048,0.025678932666778564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,8192,0.044737064838409425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,1536,0.007149866720040639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,1536,0.022834134101867676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,1536,0.013024000326792398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,1024,0.005868799984455109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,1024,0.019989333550135293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,3072,0.010712533195813497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,768,0.005892266829808554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,768,0.019259732961654664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,7168,0.04074133237202962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,512,0.005143466591835022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,512,0.01794559955596924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,6144,0.03582506577173869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,256,0.0041909332076708475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,256,0.016786134243011473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,5120,0.030355199178059893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,128,0.0037280000746250153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,128,0.015892266233762106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,3584,0.022399999698003135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,64,0.0033749334514141084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,64,0.016037333011627197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,2560,0.017794134219487508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,512,32,0.0033919999996821085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,512,32,0.015340800086657206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,2048,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,65536,0.11258347034454345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,65536,0.2984394709269206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,1024,0.010870400071144103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,65536,0.4932863871256511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,51200,0.08866666952768962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,16384,0.03526399930318196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,51200,0.2360138734181722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,16384,0.13539093335469563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,12288,0.025795199473698932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,16384,0.07960000038146972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,51200,0.39024321238199866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,10240,0.02173973321914673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,768,0.009356799721717834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,12288,0.1039189338684082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,8192,0.01658986707528432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,10240,0.08830826282501221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,8192,0.07001386483510336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,7168,0.0151637335618337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,7168,0.05964159965515137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,512,0.008539733290672303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,6144,0.013711999853452048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,256,0.007382399837176005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,6144,0.0496234655380249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,6144,0.03311786651611328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,5120,0.04094506502151489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,5120,0.027741867303848266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,4096,0.010044800241788228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,512,128,0.006816000243028005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,3584,0.009066667159398396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,3584,0.030268800258636475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,5120,0.0121888001759847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,3072,0.008437333504358928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,3072,0.0281877338886261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,4096,0.03372373183568318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,2560,0.008197333415349324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,2560,0.025830399990081788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,12288,0.06111893256505331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,10240,0.051843198140462246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,2048,0.007268266876538594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,2048,0.024126933018366496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,1536,0.006142933170000712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,8192,0.04213546514511109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,1536,0.02164586583773295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,1024,0.0052928000688552855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,7168,0.03777173360188802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,1024,0.019556266069412232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,768,0.0047199999292691554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,768,0.01802240014076233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,4096,0.02309760053952535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,512,0.004089600096146265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,512,0.01737066706021627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,3584,0.020515199502309164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,256,0.0035317334036032355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,256,0.01627626617749532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,3072,0.018332799275716148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,128,0.0033183999359607695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,2560,0.016432000199953715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,128,0.015803733468055726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,64,0.0031541332602500914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,64,0.015156267086664834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,256,32,0.0032479998966058097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,256,32,0.015144532918930054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,65536,0.10000533262888592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,2048,0.013928533593813578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,1024,0.009786666433016459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,1536,0.011702400445938111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,51200,0.07913599809010824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,768,0.00856213370958964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,16384,0.028197334210077925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,16384,0.13239893118540447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,16384,0.07779626846313477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,12288,0.023169066508611044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,12288,0.10233813126881916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,12288,0.05966399908065796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,10240,0.016295466820398966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,512,0.007478400071461995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,10240,0.08830506801605224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,8192,0.012146133184432983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,8192,0.07016106446584067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,65536,0.4874485333760579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,7168,0.011322666207949321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,7168,0.05937920014063517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,256,0.006562133133411407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,51200,0.38327999114990235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,6144,0.011554132898648579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,6144,0.04882026513417562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,5120,0.012140799562136333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,5120,0.03912746508916219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,256,128,0.005981866518656413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,5120,0.02651519974072774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,4096,0.010318932930628459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,4096,0.03288533290227254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,3584,0.009549867113431294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,3584,0.030035199721654256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,65536,0.29116799036661783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,3072,0.008923733234405517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,51200,0.22908266385396323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,3072,0.027800534168879194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,10240,0.05070399840672811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,2560,0.006977066894372304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,2560,0.025088000297546386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,8192,0.04125119845072429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,2048,0.006286933521429698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,2048,0.023332266012827556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,2048,0.013486933708190919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,1536,0.005465599894523621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,1536,0.020645334323247274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,6144,0.031233066320419313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,1024,0.004735999802748362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,7168,0.03674026727676392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,1024,0.018982400496800743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,768,0.004277333120505015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,768,0.01733013391494751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,768,0.008280533552169799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,512,0.0037621334195137024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,4096,0.02218666672706604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,512,0.017101866006851197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,256,0.0033386667569478357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,256,0.01583253343900045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,3584,0.01993173360824585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,128,0.003115733216206233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,128,0.0151637335618337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,3072,0.017577600479125977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,64,0.0029824001093705496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,64,0.015532799561818442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,128,32,0.0029824001093705496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,128,32,0.015093333522478738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,65536,0.09187946319580079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,51200,0.07331413427988688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,65536,0.48144000371297196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,51200,0.37892799377441405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,16384,0.028219733635584516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,12288,0.020588799317677816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,2560,0.01579200029373169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,16384,0.13018773396809896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,10240,0.011879466970761617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,8192,0.00978559950987498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,12288,0.09945173263549804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,7168,0.00950933297475179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,8192,0.06639573176701864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,6144,0.008728532989819845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,7168,0.0568447987238566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,5120,0.008088533580303193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,6144,0.04673386812210083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,1536,0.011390933394432068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,4096,0.007241599758466085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,3584,0.007677866518497467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,4096,0.032308266560236616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,3584,0.02919999957084656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,3072,0.02727893392244975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,2560,0.0064746667941411335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,2560,0.024954666694005333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,2048,0.005682133138179779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,2048,0.02292693257331848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,1536,0.005028266708056132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,1024,0.009543466567993163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,512,0.00738560010989507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,1024,0.004141866664091746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,768,0.0038421332836151125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,1024,0.01876693367958069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,768,0.017437867323557534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,512,0.003626666714747747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,256,0.003319466610749563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,512,0.01671253244082133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,256,0.006291200220584869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,128,0.002985599885384242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,256,0.01583466629187266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,64,0.0028746667007605235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,128,0.014891733725865683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,32,0.00290133332212766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,64,0.014855466286341348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,32,0.014872533082962037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1792,128,128,0.0058005332946777345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,10240,0.08359253406524658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,65536,0.08969919681549073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,51200,0.07058133284250895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,12288,0.01986453334490458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,16384,0.12911360263824462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,10240,0.010850133498509724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,5120,0.0379093329111735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,8192,0.009045333663622538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,51200,0.3777205467224121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,10240,0.08314666748046876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,65536,0.477346134185791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,7168,0.008803199728329976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,8192,0.06465813318888346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,6144,0.007996800045172375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,5120,0.007325866818428039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,7168,0.05481280088424682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,6144,0.04570133288701375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,5120,0.03883519967397054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,3584,0.006713599960009258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,4096,0.03142506678899129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,3072,0.0063498665889104204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,3584,0.02927253246307373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,2560,0.006916266679763794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,3072,0.026929066578547163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,2560,0.024682666858037314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,2048,0.0059797331690788266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,1536,0.004839466512203216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,2048,0.023281067609786987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,1024,0.004189866781234741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,64,1536,0.020452266931533812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,1024,0.01848533352216085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,768,0.017490132649739584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,512,0.003525333354870478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,512,0.01665386656920115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,256,0.0031541332602500914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,256,0.015666133165359496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,128,0.003005866706371307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,128,0.015148799618085226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,16384,0.025441066424051924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,64,0.002841600030660629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,32,0.002863999952872594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,64,0.014989866813023885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,32,0.014855466286341348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,12288,0.09935466448465982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,64,3072,0.0068917334079742435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,4096,0.006706133484840393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1792,32,1536,0.02023680011431376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1792,32,768,0.003824000060558319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,16384,2.472840627034505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,12288,3.3682785034179688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,12288,1.7622559865315754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,16384,4.55040537516276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,12288,2.494160970052083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,16384,3.3453600565592447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,10240,1.5192340850830077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,8192,1.1386868794759115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,10240,2.091963704427083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,10240,2.8712982177734374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,8192,2.2839391072591146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,8192,1.631104024251302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,7168,1.0746122996012368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,51200,7.90498758951823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,7168,1.4343040466308594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,7168,2.109658686319987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,6144,0.9636725107828775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,6144,1.2276084899902344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,6144,1.7098005930582683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,5120,0.7749760309855144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,5120,1.4686986287434896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,5120,0.9917023976643881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,4096,1.2082528432210287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,4096,0.6505354563395183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,4096,0.7873589197794597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,3584,1.0495317459106446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,3584,0.5511871973673503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,3072,0.48711465199788406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,3584,0.723850695292155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,3072,0.6298741022745769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,3072,0.9051541646321615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,2560,0.4182442665100098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,2560,0.7670015970865885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,2560,0.5306528091430665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,2048,0.33307199478149413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,2048,0.6020394643147786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,1536,0.46207148234049483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,2048,0.4769280115763347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,1536,0.2844778696695963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,1024,0.2119765281677246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,1536,0.38638505935668943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,1024,0.29905385971069337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,768,0.16823679606119793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,768,0.23585707346598306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,1024,0.31888427734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,512,0.1779530684153239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,512,0.15355626742045086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,768,0.2516810735066732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,256,0.13473067283630372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,512,0.21128533681233724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,256,0.1229589303334554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,128,0.10117866992950439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,256,0.20859732627868652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,128,0.10229653517405193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,128,0.1597610632578532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,64,0.0900223970413208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,32,0.08694612979888916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,64,0.10054293473561604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,65536,32,0.09959786732991537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,65536,51200,14.66982421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,65536,51200,15.168952433268228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,16384,2.1287423451741536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,16384,3.7242581685384115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,16384,2.6185226440429688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,51200,6.457865397135417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,12288,2.704180145263672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,65536,8.002350870768229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,12288,1.4815967559814454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,12288,1.9404725392659503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,10240,1.2167306264241537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,10240,2.282140858968099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,10240,1.6241269429524738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,51200,10.905741373697918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,8192,0.9687338511149088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,8192,1.8224245707194011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,51200,11.46073506673177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,8192,1.269492212931315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,7168,0.8084565480550131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,7168,1.6700939178466796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,7168,1.140007527669271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,6144,0.7276458740234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,6144,1.0002784093221029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,6144,1.420804214477539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,5120,0.6765589396158854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,5120,0.8393983840942383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,5120,1.1861695607503255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,4096,0.5100234667460124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,4096,0.678981335957845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,4096,1.0433919906616211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,65536,14.269660441080728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,3584,0.4720757484436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,3584,0.9246197382609049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,3584,0.5936245600382487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,3072,0.41501652399698896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,3072,0.6700906753540039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,2560,0.608896001180013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,65536,15.070479329427084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,2560,0.3802154541015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,2560,0.439741865793864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,2048,0.46516265869140627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,2048,0.30616639455159506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,2048,0.3745312054951986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,1536,0.3402773221333822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,1024,0.24632213910420736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,1024,0.1674239953358968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,1024,0.23033386866251626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,1536,0.3083135922749837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,768,0.18794239362080892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,1536,0.22152746518452965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,768,0.13260266780853272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,512,0.14308053652445477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,512,0.1135210673014323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,256,0.09853333632151286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,768,0.21037333806355796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,128,0.07957653204600015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,64,0.07143999735514323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,512,0.16809706687927245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,128,0.08495786984761557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,256,0.13164373238881427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,128,0.13415999412536622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,51200,32,0.0701695998509725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,64,0.08362133502960205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,32,0.08257706960042319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,51200,3072,0.513531748453776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,51200,256,0.10503253142038982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,16384,1.099294916788737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,16384,0.6355210622151692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,51200,2.11407470703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,16384,0.797930653889974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,51200,2.6632171630859376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,65536,2.8320884704589844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,12288,0.5125589370727539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,12288,0.6203178405761719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,12288,0.8335722605387369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,51200,3.5672757466634115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,10240,0.4985055923461914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,10240,0.4281130790710449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,8192,0.3648000081380208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,8192,0.6131157557169596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,10240,0.7157877604166667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,65536,3.76015370686849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,7168,0.3118624051411947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,7168,0.3580095926920573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,7168,0.48417174021403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,6144,0.2585802714029948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,8192,0.40308052698771163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,6144,0.4461514790852864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,6144,0.3213695844014486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,5120,0.2207477410634359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,5120,0.3492938677469889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,4096,0.1854464054107666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,65536,4.559415690104166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,4096,0.2696618715922038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,5120,0.2651317278544108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,4096,0.2336458683013916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,3584,0.1624501387278239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,3584,0.2545450687408447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,3584,0.20324907302856446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,3072,0.2027359962463379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,3072,0.16436692873636882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,2560,0.1275370677312215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,2560,0.1832309405008952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,2560,0.1447637399037679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,2048,0.15386026700337727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,2048,0.11623253027598064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,3072,0.1479434649149577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,2048,0.12301867008209229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,1536,0.116156800587972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,1024,0.08450773557027182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,1536,0.08881493409474692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,1536,0.1057813326517741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,1024,0.07360640366872152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,512,0.04997119903564453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,768,0.07134613196055094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,256,0.03517013390858968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,1024,0.08455039660135905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,512,0.050545068581899014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,512,0.061960534254709876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,256,0.040749867757161454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,256,0.04792319933573405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,128,0.02802986701329549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,128,0.04579093456268311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,64,0.02573440074920654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,128,0.038583465417226154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,64,0.03884160121281942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,16384,32,0.025882667303085326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,32,0.03964373270670573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,16384,768,0.06033600171407064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,16384,768,0.0673472007115682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,51200,1.5952351888020835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,16384,0.796671994527181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,65536,2.023352559407552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,16384,0.5176501274108887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,51200,2.7338528951009113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,65536,2.713156382242839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,16384,0.6353141148885091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,12288,0.627078374226888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,12288,0.38928000132242835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,10240,0.34665921529134114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,51200,1.9763786315917968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,8192,0.38979838689168295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,65536,3.4765055338541666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,10240,0.38885974884033203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,12288,0.48081385294596357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,10240,0.54956480662028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,8192,0.2563594659169515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,8192,0.3182954788208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,7168,0.24003626505533854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,7168,0.4010165214538574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,6144,0.21303466161092124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,5120,0.2503904024759928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,6144,0.25683199564615883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,5120,0.17797013918558757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,7168,0.3027114550272624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,5120,0.2047914663950602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,4096,0.14818026224772135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,4096,0.16854826609293622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,3584,0.18157226244608562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,3584,0.1300597349802653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,3584,0.1580885410308838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,3072,0.11814080079396565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,2560,0.0994815985361735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,2048,0.10866666634877523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,2048,0.08495679696400961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,2048,0.09617066383361816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,1536,0.08290133476257325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,1536,0.06938666502634684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,1536,0.07672106424967448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,1024,0.05975040197372437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,1024,0.054586664835611975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,1024,0.05880213181177775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,768,0.04749759833017985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,768,0.04720853169759114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,768,0.05333120028177897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,512,0.03629333178202311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,512,0.04020373423894246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,512,0.043953065077463785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,6144,0.2976223945617676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,256,0.033583998680114746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,128,0.020386133591334024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,256,0.03775786558787028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,128,0.031098665793736775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,128,0.0353493332862854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,64,0.01799466609954834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,4096,0.23600533803304038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,32,0.01811199982961019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,64,0.031514666477839154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,3072,0.15311892827351886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,12288,32,0.031228800614674885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,2560,0.13437333106994628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,2560,0.11323413054148357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,12288,256,0.02553386688232422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,12288,3072,0.13032960096995036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,51200,1.3834261576334634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,16384,0.6674837112426758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,51200,1.6023508707682292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,65536,1.7943135579427083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,16384,0.44513918558756516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,16384,0.5105429331461588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,12288,0.5675658543904623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,65536,2.216596221923828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,12288,0.3513909339904785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,12288,0.39410346349080405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,10240,0.4213216145833333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,10240,0.3029077212015788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,10240,0.34651947021484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,8192,0.2359861373901367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,8192,0.3831637382507324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,8192,0.26549866994222004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,7168,0.31640640894571936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,65536,3.001819610595703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,7168,0.21110933621724448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,7168,0.23641279538472496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,6144,0.26226879755655924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,6144,0.19150293668111165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,6144,0.20597759882609049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,5120,0.2391530672709147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,5120,0.16744106610616047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,4096,0.17825279235839844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,4096,0.12986986637115477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,5120,0.17967467308044432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,4096,0.1453706741333008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,3584,0.17241066296895344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,3584,0.11537280082702636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,3072,0.1372085412343343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,3072,0.10728533267974853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,3584,0.12679680188496906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,3072,0.1132309357325236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,2560,0.12803520361582438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,2560,0.090010666847229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,2048,0.09475306669871011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,2560,0.09787519772847494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,2048,0.0778592030207316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,2048,0.08198933601379395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,1536,0.06313066482543946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,1024,0.05459200143814087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,51200,2.26986567179362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,1024,0.051267198721567785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,768,0.041857067743937174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,1536,0.06477226813634238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,1024,0.05415680011113485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,512,0.032536532481511435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,768,0.0447648008664449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,256,0.023774933815002442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,512,0.037256534894307455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,128,0.018772266308466592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,256,0.03093973398208618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,256,0.033019733428955075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,128,0.028463999430338543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,64,0.01498133341471354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,128,0.030663466453552245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,32,0.015568000078201295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,64,0.030324266354242964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,10240,32,0.02893120050430298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,10240,1536,0.08161280155181885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,512,0.037161600589752194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,10240,768,0.04435840050379435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,65536,1.4948394775390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,51200,1.812979253133138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,16384,0.5466741561889649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,16384,0.39148906071980794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,65536,2.382456461588542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,51200,1.1680277506510417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,12288,0.41582721074422196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,12288,0.2991872151692708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,10240,0.26160426139831544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,10240,0.36033493677775064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,65536,2.7708287556966145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,8192,0.27924585342407227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,12288,0.4066015879313151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,8192,0.2098325411478678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,7168,0.1818079948425293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,7168,0.2571200052897135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,6144,0.2141333262125651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,7168,0.25653120676676433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,6144,0.15537385940551757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,5120,0.1791754722595215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,5120,0.13214720090230306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,4096,0.1464906692504883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,4096,0.10929706891377766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,3584,0.12552213668823242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,3584,0.09854079882303873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,16384,0.562608019510905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,3072,0.11330773035685222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,10240,0.34291839599609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,3072,0.09296320279439291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,2560,0.09644693533579508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,8192,0.2822410583496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,2560,0.08128000100453694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,2048,0.07916906674702963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,51200,1.9541044871012372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,2048,0.06663360198338827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,1536,0.06081706682840983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,5120,0.17780267397562663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,1536,0.054870398839314785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,6144,0.21261866887410483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,1024,0.043347199757893876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,4096,0.14708800315856935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,1024,0.04466986656188965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,768,0.0351744016011556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,768,0.03930346568425496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,512,0.026366933186848955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,512,0.03362239996592204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,512,0.028479999303817748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,256,0.018873600165049235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,256,0.028716800610224406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,256,0.02373973329861959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,128,0.01472106675306956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,128,0.0269375999768575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,2048,0.08632000287373862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,64,0.012707199652989706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,64,0.026828799645106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,8192,32,0.013136000434557597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,8192,32,0.026757333676020307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,3584,0.12468799750010173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,3072,0.10342613061269124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,2560,0.09228053092956542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,1536,0.0547327995300293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,768,0.03422506650288899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,1024,0.043339733282725015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,65536,2.0305482228597005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,65536,1.285132853190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,16384,0.3510186513264974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,16384,0.511896546681722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,8192,128,0.019964800278345744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,51200,0.9890111923217774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,12288,0.250761604309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,12288,0.3619722684224447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,51200,1.4290475209554037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,12288,0.36604372660319007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,10240,0.2812106768290202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,10240,0.23356159528096518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,8192,0.22790613174438476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,51200,1.6806143442789714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,65536,2.3449076334635417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,8192,0.17310400009155275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,7168,0.2010218620300293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,8192,0.24949866930643716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,6144,0.17110613187154133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,7168,0.17145172754923504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,6144,0.14091626803080243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,5120,0.14522560437520343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,5120,0.13297066688537598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,4096,0.11866346995035808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,4096,0.09862720171610515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,3584,0.10405120054880779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,3584,0.08846080303192139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,3584,0.11037866274515788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,3072,0.09037439823150635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,3072,0.0774346669514974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,3072,0.0933247963587443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,2560,0.07648959954579672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,10240,0.31531092325846355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,2560,0.06891733010609945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,2560,0.08243626753489176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,2048,0.06195946534474691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,2048,0.059154133001963295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,2048,0.063482666015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,1536,0.04846400022506714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,1536,0.04927146832148234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,1024,0.03532693386077881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,1536,0.04796160062154134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,1024,0.04100159804026286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,768,0.02919466694196065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,1024,0.038650667667388915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,768,0.035665067036946614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,512,0.022490666309992472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,768,0.030510934193929036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,512,0.025150932868321735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,256,0.015924266974131265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,512,0.031128533681233722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,128,0.012584533294041952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,256,0.026715733607610065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,128,0.024270933866500855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,128,0.01785279909769694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,64,0.011226666967074077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,64,0.025026132663091023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,7168,32,0.024286933739980063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,16384,0.4989269256591797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,7168,0.22255360285441078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,6144,0.18736960093180338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,5120,0.15584959983825683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,7168,32,0.011482666929562886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,65536,1.158519490559896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,65536,1.6733706156412762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,4096,0.13072319825490314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,16384,0.38095146814982095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,16384,0.31501439412434895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,16384,0.33266239166259765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,12288,0.3134709358215332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,51200,0.9041023890177409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,12288,0.2344053268432617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,51200,1.0223050435384116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,12288,0.2472501277923584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,51200,1.2604170481363932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,10240,0.19959146181742352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,10240,0.2635658740997314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,10240,0.21053226788838705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,8192,0.15607679684956868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,8192,0.23172373771667482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,7168,0.1733791987101237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,7168,0.15076053937276204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,8192,0.1866901397705078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,6144,0.14730772972106934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,7168,0.15149653752644857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,7168,256,0.020906666914621987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,6144,0.1330624024073283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,5120,0.12708693345387775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,5120,0.11410346825917561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,4096,0.1003989299138387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,3584,0.09127039909362793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,4096,0.09294079939524333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,3584,0.08189333279927571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,3584,0.08204692999521891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,3072,0.07851839860280355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,3072,0.07195946375528971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,2560,0.06522453228632609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,2560,0.06164906819661459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,2560,0.06201279958089193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,2048,0.05284693241119385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,2048,0.052875733375549315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,2048,0.05336533387502035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,1536,0.041606398423512776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,1536,0.044447998205820724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,1024,0.030123732487360638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,1024,0.03512426614761353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,1024,0.033890132109324136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,768,0.02427840034166972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,65536,1.340719985961914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,768,0.030719999472300214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,512,0.01872533361117045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,768,0.030741333961486816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,512,0.02704106569290161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,512,0.025934932629267375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,256,0.013185066978136697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,256,0.023931733767191567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,256,0.020948266983032225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,128,0.00998293360074361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,6144,0.13991039594014484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,128,0.019553067286809285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,64,0.008587732911109924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,128,0.021316266059875487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,6144,32,0.008961066603660583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,4096,0.08382613658905029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,64,0.02160853346188863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,6144,32,0.021203200022379555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,51200,1.0481365203857422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,65536,1.0353301366170249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,65536,1.063644790649414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,65536,1.4027936299641928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,16384,0.33282454808553064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,5120,0.10948692957560222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,16384,0.30106986363728844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,16384,0.2984085400899251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,12288,0.24425172805786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,51200,0.8330560048421225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,51200,0.8690250396728516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,12288,0.22081173261006676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,10240,0.20414719581604004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,10240,0.16948053042093914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,10240,0.18133759498596191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,8192,0.14927466710408527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,8192,0.19310933748881023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,7168,0.14335467020670573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,7168,0.12919680277506512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,7168,0.13454933166503907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,6144,0.14683732986450196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,6144,0.10353279908498128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,6144,0.11066239674886066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,5120,0.10816000302632649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,5120,0.10434133211771648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,4096,0.08429333368937174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,5120,0.09169066747029622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,4096,0.07975680033365885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,3584,0.07446293036142984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,4096,0.08173759778340658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,3584,0.06990293661753336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,3584,0.07037973403930664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,3072,0.06856319904327393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,2560,0.05609386761983236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,2560,0.05702933470408121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,2560,0.052697598934173584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,2048,0.04587626854578654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,2048,0.047595731417338055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,2048,0.04648533264795939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,1536,0.035854931672414145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,1536,0.03971946636835734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,1536,0.03699306646982829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,1024,0.025676800807317095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,1024,0.03181119958559672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,768,0.020787199338277183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,1024,0.030100266138712566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,768,0.029123200972874956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,512,0.016218666235605875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,512,0.02513386607170105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,8192,0.13955519994099935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,512,0.022190932432810465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,256,0.011217066645622253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,256,0.021661865711212158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,128,0.008609066406885784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,128,0.02001919945081075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,128,0.01676266590754191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,64,0.007428266604741414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,64,0.02073919971783956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,32,0.007897600034872691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,5120,3072,0.06467519998550415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,5120,32,0.020425599813461304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,768,0.02663573424021403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,256,0.017925333976745606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,65536,1.0251328150431314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,65536,0.9079551696777344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,51200,0.7182826360066732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,51200,0.8199509302775064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,16384,0.2609290599822998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,16384,0.2477344036102295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,1536,0.04188906749089559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,12288,0.1966869354248047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,12288,0.17778239250183106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,6144,3072,0.07027520338694254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,10240,0.16879466374715169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,10240,0.15158185958862305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,12288,0.202787192662557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,8192,0.1339626630147298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,8192,0.12011946837107342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,7168,0.117031462987264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,7168,0.10601066748301188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,5120,3072,0.05961066484451294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,6144,0.1004640022913615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,6144,0.09429866472880045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,65536,1.3749109903971353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,5120,0.09495786825815836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,16384,0.32177279790242513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,5120,0.08979093233744304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,4096,0.07059946854909262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,4096,0.07292693456013997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,51200,1.0450464248657227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,3584,0.06895679632822672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,3584,0.07008319695790609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,12288,0.24079893430074056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,3072,0.052642134825388584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,3072,0.05369706551233927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,10240,0.19718720118204752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,2560,0.0452565352121989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,2560,0.04770986636479695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,7168,0.1440384070078532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,2048,0.03646399974822998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,2048,0.04031039873758952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,8192,0.15943360328674316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,1536,0.028148265679677327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,1536,0.03420053323109944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,6144,0.12363733450571697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,1024,0.020593067010243736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,1024,0.028794666131337483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,3072,0.062097068627675375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,768,0.01717653274536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,768,0.02541866699854533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,4096,0.08162240187327066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,512,0.013436800241470337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,512,0.023602133989334105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,5120,0.10288960138956707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,256,0.009274666508038838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,256,0.02054506738980611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,3584,0.07166186968485513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,128,0.007673599819342296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,128,0.01928640007972717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,2560,0.05279039939244588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,64,0.006746666630109151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,64,0.019668267170588175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,4096,32,0.007165866593519847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,4096,32,0.019875200589497884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,1536,0.03172053297360738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,65536,0.95119997660319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,65536,0.8409706751505533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,1024,0.025169066588083905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,51200,0.732371203104655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,2048,0.04113813241322835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,16384,0.22938240369160973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,51200,0.651137097676595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,51200,0.8064362843831381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,16384,0.21808320681254068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,12288,0.18953812917073568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,12288,0.16523946126302083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,10240,0.1453525384267171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,12288,0.18655680020650228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,10240,0.13722880681355792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,8192,0.1170570691426595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,8192,0.1147648016611735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,8192,0.13264426390329997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,256,0.013860266407330832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,512,0.017349332571029663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,7168,0.1031765302022298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,7168,0.09823359648386637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,7168,0.12043519814809163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,128,0.012427733341852824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,6144,0.08884159723917642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,6144,0.08672746817270914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,5120,0.07480533123016357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,5120,0.07643199761708577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,5120,0.0842357317606608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,4096,0.06078293323516846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,4096,0.0634442647298177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,3584,0.053388798236846925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,4096,0.07103253205617269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,3584,0.05704746643702189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,3584,0.05928639968236288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,3072,0.04976426760355632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,3072,0.05006826718648275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,2560,0.03864320119222005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,2560,0.04370133479436238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,2560,0.044334932168324785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,2048,0.031245867411295574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,4096,768,0.02108373244603475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,2048,0.03654293219248454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,1536,0.02412373423576355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,1536,0.031052800019582112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,65536,1.0184789021809897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,1024,0.02125119964281718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,1024,0.030178133646647138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,1024,0.02655893365542094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,768,0.01748266617457072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,768,0.026945066452026368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,768,0.021910399198532104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,512,0.012922666470209756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,3072,0.0449621319770813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,512,0.023703465859095253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,16384,0.25875733693440756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,256,0.00825386643409729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,256,0.013806933164596557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,256,0.02026453415552775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,128,0.00788373351097107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,128,0.01964799960454305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,128,0.011648000280062357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,64,0.006749866902828217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,2048,0.03537493149439494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3584,32,0.008609066406885784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,64,0.020434133211771646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3584,32,0.020269866784413657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,10240,0.16375999450683593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,65536,0.7627861022949218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,65536,0.7677141189575195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,51200,0.5885941187540691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,6144,0.0991541306177775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,16384,0.19131627082824706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,16384,0.1999743938446045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,51200,0.648134422302246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,12288,0.14400854110717773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,12288,0.1556447982788086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,10240,0.11878399848937989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,1536,0.027165865898132323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,10240,0.12998080253601074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,8192,0.09604053497314453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,8192,0.10209706624348958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,7168,0.08556693394978841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,7168,0.09092373053232829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3584,512,0.015185067057609558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,6144,0.07249173323313395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,6144,0.07981653213500976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,65536,0.9918656031290689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,5120,0.06592533191045126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,5120,0.07761387030283609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,16384,0.24247466723124184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,4096,0.05069973468780518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,51200,0.777894401550293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,4096,0.05918399890263876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,3584,0.044870400428771974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,12288,0.1847658634185791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,3584,0.05296853383382162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,3584,0.05544960101445516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,3072,0.039212799072265624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,3072,0.046189868450164796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,2560,0.033242666721343996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,2560,0.040423464775085446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,8192,0.12507839997609455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,2048,0.027497599522272747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,10240,0.16033066113789876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,2048,0.03546773195266724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,1536,0.021975467602411904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,1536,0.030602665742238362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,7168,0.1108394702275594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,1024,0.01649173299471537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,1024,0.025466666618982954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,6144,0.09540266990661621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,768,0.013697066903114319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,768,0.023931733767191567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,5120,0.07798826694488525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,512,0.010763733585675558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,512,0.021203200022379555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,4096,0.06494826475779215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,256,0.008121599753697712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,256,0.01917546590169271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,3072,0.04779306650161743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,128,0.006712533533573151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,128,0.018283732732137046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,2560,0.0414463996887207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,64,0.005894400179386139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,64,0.018359466393788656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,3072,32,0.006205866734186808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,3072,32,0.01830186645189921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,2048,0.03278719981511434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,1536,0.0252895991007487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,1024,0.020168532927831016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,768,0.016979199647903443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,65536,0.6355349222819011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,65536,0.689569091796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,51200,0.4979082743326823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,16384,0.15986026128133138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,51200,0.5394805272420247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,16384,0.19321707089742024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,16384,0.15306986172993978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,12288,0.11935573418935139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,12288,0.14672746658325195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,10240,0.10031147003173828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,12288,0.12359466552734374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,10240,0.12046293417612712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,8192,0.08183786869049073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,10240,0.10024320284525554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,8192,0.09619413216908773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,7168,0.07280639807383218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,7168,0.08512319723765055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,7168,0.07456639607747396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,6144,0.06282773415247599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,512,0.014196266730626425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,6144,0.07520853678385417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,6144,0.06336106856664023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,5120,0.05300480127334595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,5120,0.05430506865183512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,5120,0.06433279911677042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,4096,0.043110398451487224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,256,0.011785599589347839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,4096,0.05399146477381388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,4096,0.04493653376897176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,3584,0.03862186670303345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,51200,0.49294398625691727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,3584,0.04756266673405965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,3584,0.040608000755310056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,3072,0.03350293238957723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,3072,0.0351146658261617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,2560,0.02823999921480815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,3072,0.04224640130996704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,2048,0.02349546750386556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,2560,0.037690667311350505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,3072,128,0.010501333077748616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,1536,0.01922453244527181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,2048,0.026387200752894087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,2048,0.03324693242708842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,1536,0.028683733940124512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,1024,0.014333867033322654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,1536,0.02250666618347168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,1024,0.023745065927505492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,768,0.012012799580891926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,512,0.009351467092831928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,768,0.015999999642372132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,512,0.01404800017674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,512,0.02072319984436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,256,0.007149866720040639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,256,0.018644267320632936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,256,0.011705600221951802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,128,0.00602346658706665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,128,0.018151466051737467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,64,0.0052704001466433205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,64,0.017746132612228394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2560,32,0.005628799895445505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,32,0.018261333306630455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,65536,0.5248277346293132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2560,768,0.022648533185323082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,128,0.010936533411343891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,65536,0.6487744013468425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,51200,0.40712534586588545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,65536,0.6955370585123698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,16384,0.1427072048187256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,51200,0.5027381261189778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,16384,0.17890453338623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,51200,0.5720309575398763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,12288,0.09978880087534586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,12288,0.13330453236897785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,10240,0.08397440115610758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,10240,0.11287786960601806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,8192,0.06874667008717855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,8192,0.0904362678527832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,16384,0.17556479771931965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,7168,0.05964266856511434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,10240,0.11231679916381836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,12288,0.13354239463806153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,7168,0.0784277359644572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,6144,0.05094826618830363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,7168,0.08056853612263998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,6144,0.06900479793548583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,5120,0.04347840150197347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,65536,0.5887093226114909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,5120,0.05890239874521891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,4096,0.03889600038528442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,4096,0.047591467698415116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,8192,0.08859519958496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,3584,0.031301333506902056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,3584,0.043108268578847246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,3072,0.02762666742006938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,3072,0.03851840098698934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,3072,0.035043199857076004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,2560,0.023887999852498374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,2560,0.03424959977467855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,2560,0.0304639995098114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,2048,0.01931519905726115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,2048,0.029882667462031047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,8192,0.08149973551432291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,1536,0.015846400459607442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,1536,0.025896533330281572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,1536,0.019796266158421835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,1024,0.011818666259447734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,1024,0.02274880011876424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,5120,0.05776960055033366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,6144,0.06714133421579996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,768,0.009819733103116353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,768,0.02132800022761027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,512,0.007956266899903615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,4096,0.047421868642171225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,3584,0.04055893421173096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,512,0.019182932376861573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2560,1024,0.017516799767812095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,256,0.006231466432412466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,512,0.011634133259455363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,128,0.005314133564631144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,256,0.009417600433031718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,256,0.018216532468795777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,64,0.004835199813048045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,128,0.016797866423924765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,2048,32,0.005120000243186951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,64,0.01745599905649821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,2048,32,0.016999467213948568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,2560,0.031109333038330078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,2048,0.02474773327509562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,65536,0.39147841135660805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,65536,0.6038858413696289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,51200,0.30494934717814126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,16384,0.11994240283966065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,51200,0.4782037417093913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,16384,0.16144533157348634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,12288,0.09086613655090332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,12288,0.12351573308308919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,1024,0.015589333573977151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,10240,0.07725653648376465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,10240,0.10511039892832438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,768,0.013543466726938883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,8192,0.05995093186696371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,8192,0.08559359709421793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,7168,0.0533023993174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,2048,128,0.008450133601824443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,7168,0.07644266287485758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,6144,0.047014399369557695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,6144,0.06622080008188883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,12288,0.12489813168843586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,5120,0.039640533924102786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,5120,0.05562133391698202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,65536,0.6530954360961914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,16384,0.16808853149414063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,4096,0.03220053315162659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,4096,0.045210667451222736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,3584,0.02872320016225179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,3584,0.041681067148844404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,51200,0.5115797360738118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,3072,0.025034666061401367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,3072,0.03603839874267578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,10240,0.10732693672180176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,2560,0.021425066391626994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,6144,0.06257280111312866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,2560,0.03218239943186442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,2048,0.01790720025698344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,2048,0.028523733218510944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,7168,0.07210559844970703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,1536,0.014387200276056925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,1536,0.02373759945233663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,8192,0.07976853052775065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,1024,0.010627200206120808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,1024,0.021160533030827842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,3072,0.03222506642341614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,768,0.008850133419036866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,4096,0.04378773371378581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,768,0.020232532421747842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,512,0.007167999943097432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,512,0.018730666240056357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,5120,0.05148799816767374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,256,0.005469866593678792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,256,0.017233065764109292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,3584,0.036697598298390706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,128,0.004760533571243286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,2560,0.027769599358240766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,128,0.016385066509246825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,64,0.004315733412901561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,64,0.01653333306312561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1536,32,0.004430933296680451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1536,32,0.016986666123072307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,1536,0.01722453236579895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,65536,0.26887146631876624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,2048,0.022342399756113688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,65536,0.5170624097188313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,51200,0.20766827265421547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,1024,0.014363732933998109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,51200,0.4098623911539714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,16384,0.07071680227915446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,16384,0.13632213274637858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,12288,0.053939199447631835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,12288,0.10547306537628173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,65536,0.4736949284871419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,10240,0.04616853396097819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,10240,0.0904970645904541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,51200,0.3696330706278483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,16384,0.12489066918690998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,8192,0.03656959931055705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,128,0.00740479975938797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,8192,0.07291413148244222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,7168,0.03294613361358643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,256,0.008203733464082081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,7168,0.06386880079905191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,6144,0.028279467423756914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,512,0.010125866532325745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,6144,0.05440746545791626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,5120,0.023730132977167764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,12288,0.09379093647003174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,5120,0.04549546639124553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,4096,0.019413334131240845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,5120,0.04132906595865886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,4096,0.03730346759160359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,3584,0.017576533555984496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,3584,0.03410559892654419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1536,768,0.011827199657758077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,3072,0.01594239970048269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,3072,0.031490133206049604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,3072,0.02704746723175049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,2560,0.014045866330464682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,2560,0.028284800052642823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,10240,0.07994133631388346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,2048,0.011633066336313884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,2048,0.025197867552439374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,8192,0.0670261303583781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,1536,0.009499733646710713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,1536,0.02249173323313395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,7168,0.05749653180440267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,1536,0.015685333808263143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,1024,0.007439999779065449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,1024,0.020001065731048585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,768,0.006584533552328746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,6144,0.05024853150049845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,768,0.01950613260269165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,512,0.00565226674079895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,512,0.01769386728604635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,4096,0.03415786822636922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,256,0.004667733112970988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,3584,0.030580266316731768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,256,0.0167797327041626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,128,0.00420906643072764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,128,0.016497066617012023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,64,0.003908266623814901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,64,0.016105600198109946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,1024,32,0.004212266703446706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,1024,32,0.016119466225306193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,65536,0.20684587160746254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,2560,0.023093332846959434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,65536,0.48627519607543945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,51200,0.16476267178853352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,51200,0.38184213638305664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,2048,0.019393066565195717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,16384,0.07404800256093344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,1024,0.011854933698972066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,16384,0.14088212649027507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,12288,0.05744426647822062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,768,0.01018986701965332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,16384,0.11725013256072998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,12288,0.11046400070190429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,512,0.008918399612108868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,10240,0.09375999768575033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,8192,0.03829013506571452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,256,0.007179733117421467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,8192,0.07438720067342122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,7168,0.034746666749318436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,8192,0.061861332257588705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,1024,128,0.00658240020275116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,7168,0.06452800035476684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,6144,0.03025173346201579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,6144,0.05504746834437052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,5120,0.025921066602071125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,5120,0.04592853387196859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,10240,0.047707732518514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,65536,0.44389867782592773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,4096,0.020973867177963255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,4096,0.038771200180053714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,3584,0.01857173244158427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,3584,0.0343178669611613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,51200,0.3399125417073568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,3072,0.01662506659825643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,3072,0.03129173318545024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,12288,0.08771093686421713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,2560,0.014481066664059957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,10240,0.0746122678120931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,2560,0.028436267375946046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,2048,0.012152533729871113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,2048,0.025409066677093507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,7168,0.05363839864730835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,1536,0.009655466675758362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,6144,0.0460970679918925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,1536,0.02301973303159078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,1024,0.0075914666056633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,1024,0.019977599382400513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,5120,0.0392576018969218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,1024,0.01125866671403249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,768,0.006647466619809468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,768,0.018959999084472656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,512,0.00543146679798762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,512,0.017498666048049928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,4096,0.03231253425280253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,256,0.0043722664316495265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,256,0.016521599888801575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,3584,0.0284223993619283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,128,0.0038677332301934562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,128,0.015556266903877259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,3072,0.024091732501983643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,64,0.0035391998787721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,2560,0.02135573426882426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,64,0.01597866714000702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,2048,0.01808213392893473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,1536,0.01423893372217814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,768,32,0.01590506633122762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,65536,0.14225600560506185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,768,0.00941439966360728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,512,0.008168533444404602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,51200,0.11487360000610351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,65536,0.4518783887227376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,16384,0.04262719949086507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,16384,0.12430400053660076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,51200,0.35556160608927406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,12288,0.03276693423589071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,768,32,0.003734400123357773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,256,0.006572799881299336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,12288,0.09655573368072509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,10240,0.028702932596206664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,768,128,0.005966933568318685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,8192,0.02176533341407776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,8192,0.06377280155817668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,7168,0.01927466591199239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,7168,0.05510400136311849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,10240,0.08147733211517334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,6144,0.016531200210253397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,6144,0.0464842677116394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,12288,0.08142399787902832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,5120,0.015130666891733804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,5120,0.038182401657104494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,65536,0.40610561370849607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,4096,0.011923199892044068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,16384,0.10706559816996257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,4096,0.03226666649182637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,3584,0.010540800293286641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,3584,0.02960639993349711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,10240,0.06862080097198486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,51200,0.31495787302652994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,3072,0.009970133503278095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,3072,0.02785173257191976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,2560,0.00851200024286906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,2560,0.02513813376426697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,8192,0.05625919898351034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,2048,0.007490133245786031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,7168,0.04857493241628011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,2048,0.023013333479563393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,1536,0.006687999765078227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,1536,0.020578134059906005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,6144,0.04248426755269368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,1024,0.005491200089454651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,1024,0.01993173360824585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,3072,0.02297280033429464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,768,0.004890666902065277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,768,0.01858133276303609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,2560,0.01981226603190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,1536,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,512,0.004322133461634318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,5120,0.03658026854197184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,256,0.003918933371702829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,512,0.017324799299240114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,4096,0.029813333352406816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,256,0.0165503998597463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,128,0.0035082665582497918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,128,0.015476266543070475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,64,0.003335466732581457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,64,0.015379200379053751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,512,32,0.003538133452335993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,3584,0.02585279941558838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,512,32,0.01532373329003652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,65536,0.09381972948710124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,2048,0.01643946667512258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,65536,0.42625172932942706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,51200,0.07658773263295492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,51200,0.21702720324198405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,51200,0.33828585942586265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,16384,0.030777599414189654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,16384,0.11919360160827637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,12288,0.02495786746342977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,12288,0.09203306833902994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,1024,0.010452266534169514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,10240,0.020713599522908528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,10240,0.07698773543039958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,10240,0.04834880034128825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,8192,0.017767467101415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,8192,0.0597055991490682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,65536,0.27054932912190754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,7168,0.016936532656351724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,7168,0.050261334578196204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,7168,0.03402026494344075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,6144,0.014773333072662353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,6144,0.04299733241399129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,6144,0.029865600665410358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,5120,0.011966933806737263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,12288,0.05567786693572998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,5120,0.025604265928268432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,5120,0.03540053367614746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,256,0.00631039987007777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,4096,0.01037440001964569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,3584,0.00960533320903778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,4096,0.03049066662788391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,3584,0.029046400388081865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,768,0.008836266398429871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,512,0.007705600063006084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,3072,0.008494933446248371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,3072,0.026796799898147584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,2560,0.008082133531570435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,8192,0.03911573489507039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,2560,0.024513065814971924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,2048,0.006743466854095459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,2560,0.015524267156918844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,2048,0.02259520093599955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,1536,0.006161066889762879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,16384,0.07728959719340006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,512,128,0.0058218667904535925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,1024,0.005008000135421753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,1024,0.01857173244158427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,1536,0.02131839990615845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,768,0.004411733150482178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,768,0.017640533049901326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,4096,0.02140373388926188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,512,0.003976533313592275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,3584,0.019205333789189656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,512,0.016771199305852254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,256,0.0035264000296592714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,3072,0.017293866475423178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,256,0.015713066856066386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,256,0.006381866832574208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,128,0.00329066663980484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,64,0.003101866692304611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,128,0.005927466849486033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,256,32,0.003068800022204717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,64,0.015122133493423461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,32,0.015210666259129844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,2048,0.013337600231170654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,65536,0.08342400391896566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,51200,0.06490879853566488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,65536,0.41633812586466473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,51200,0.32882026036580403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,16384,0.026371200879414875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,51200,0.20535359382629395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,1024,0.009086933732032777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,16384,0.11668480237325032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,12288,0.018210132916768394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,12288,0.09039893150329589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,10240,0.014643200238545737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,10240,0.07528639634450277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,256,128,0.015250133474667868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,10240,0.04564479986826579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,8192,0.012217600146929424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,8192,0.057351466019948324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,7168,0.013697066903114319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,1536,0.0118559996287028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,7168,0.04838826656341553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,6144,0.012292266885439555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,768,0.008177066842714947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,6144,0.04145173231760661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,5120,0.010660266876220703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,5120,0.034893866380055746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,5120,0.024331732590993246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,4096,0.009360000491142273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,4096,0.029925332466761274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,4096,0.020466132958730062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,3584,0.0084906667470932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,3584,0.02841599980990092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,256,512,0.007430399954319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,3072,0.0075914666056633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,3072,0.025933865706125898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,3072,0.01634666621685028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,2560,0.006785066425800323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,2560,0.023290665944417317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,65536,0.2634165287017822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,2048,0.006277333199977875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,2048,0.022141865889231362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,16384,0.07008319695790609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,1536,0.005313066641489664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,1536,0.02032533288002014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,12288,0.05407253503799438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,1024,0.004586666822433472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,1024,0.018155733744303383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,1024,0.008742400010426839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,768,0.004138666639725367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,768,0.01817173361778259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,8192,0.037426133950551346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,768,0.007893333335717519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,512,0.00373333344856898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,512,0.01662826637427012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,512,0.007337599992752075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,7168,0.032858665784200033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,256,0.003387733300526937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,128,0.003053866575161616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,256,0.015759999553362526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,128,0.015201066931088766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,6144,0.02873493234316508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,64,0.002919466545184453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,128,32,0.0029546665648619336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,64,0.014884266257286071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,128,32,0.015254400173823037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,65536,0.07964373429616292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,51200,0.06263466676076254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,65536,0.415666135152181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,16384,0.024401066700617473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,3584,0.01882773240407308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,51200,0.3275957425435384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,12288,0.014317867159843446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,16384,0.11389013131459554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,12288,0.08732586701711019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,10240,0.01111466685930888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,8192,0.010730666915575664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,10240,0.0720970630645752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,7168,0.010185600320498148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,8192,0.055276799201965335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,6144,0.009373866518338521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,7168,0.04706133206685384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,5120,0.008573866883913676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,6144,0.03993813196818034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,4096,0.008477866649627686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,5120,0.03334186474482219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,3584,0.007916800181070964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,4096,0.029202133417129517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,3072,0.0070613334576288865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,3072,0.025256532430648803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,2560,0.00636053333679835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,2560,0.01461120049158732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,2560,0.023982934157053628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,2048,0.00555733342965444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,1536,0.004841599861780802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,2048,0.021848533550898233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,2048,0.012658133109410604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,1024,0.004277333120505015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,1536,0.019820799430211387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,768,0.003918933371702829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,1024,0.01885973413785299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,512,0.0035487999518712364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,768,0.017383466164271034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,256,0.0031818665564060213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,512,0.01632426679134369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,128,0.0028885332246621448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,256,0.015449600418408713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,64,0.0028277332584063213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,128,0.014859732985496522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,64,32,0.0028959999481836954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,64,0.01471573313077291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,32,0.014882133404413859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,1536,0.010964266459147136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,51200,0.06433813174565634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,65536,0.08113919893900554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,16384,0.02444480061531067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,12288,0.01267306705315908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,16384,0.11285866896311443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,10240,0.01134399970372518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,65536,0.4136362711588542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,51200,0.3258421262105306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,12288,0.0860042651494344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,8192,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,10240,0.07127040227254232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,6144,0.008984532952308655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,8192,0.05354453325271606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,7168,0.04537599881490072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,5120,0.007774933179219563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,4096,0.00691840002934138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,256,0.006132266422112783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,5120,0.03349333206812541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,3584,0.00717439999183019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,4096,0.02832319935162862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,3584,0.027108265956242876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,3072,0.006417066852251689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,2560,0.006123733520507812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1537,128,128,0.005726933479309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,3072,0.025548799832661943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,2048,0.005430399874846141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,2560,0.023693867524464927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,1536,0.00477866679430008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,1024,0.004216533402601878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,1536,0.01991573373476664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,768,0.0038592000802357995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,1024,0.017889066537221273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,768,0.017142399152119955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,512,0.0035157332817713416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,256,0.0031360000371932983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,512,0.016293332974116007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,128,0.0029578665892283124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,256,0.015413332978884378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,64,0.00278613343834877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,128,0.01497706671555837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,32,0.002926933268706004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,64,0.014862933754920959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,32,0.014501333236694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1537,32,7168,0.009546666344006857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,64,3584,0.027738666534423827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,6144,0.03864746491114299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1537,32,2048,0.021997867027918498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,12288,1.7204811096191406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,16384,2.3452959696451825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,16384,2.9322357177734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,12288,3.4591616312662756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,16384,4.5454869588216145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,10240,1.4187455495198569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,12288,2.2219520568847657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,10240,2.7707242329915363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,10240,1.8947072347005207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,8192,1.2244661966959636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,8192,1.5152533213297525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,8192,2.218396759033203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,7168,1.1043317159016928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,51200,7.2654378255208325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,7168,1.924951426188151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,7168,1.3026859283447265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,6144,0.9419306437174478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,6144,1.6656213124593098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,6144,1.0351733525594076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,5120,0.8028533299763998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,5120,1.398312505086263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,5120,0.9693621317545572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,4096,0.5945536295572917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,4096,1.101971181233724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,51200,10.271260579427082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,3584,0.5483861287434896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,4096,0.735430399576823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,3584,0.9657503763834635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,3584,0.6146303812662761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,3072,0.46202665964762374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,3072,0.8486421585083008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,2560,0.41478827794392903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,2048,0.3224149386088053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,2560,0.5003861427307129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,3072,0.5919509251912435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,2560,0.6967679977416992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,2048,0.5687007904052734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,2048,0.4932405471801758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,1536,0.41205441157023115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,1536,0.28432426452636717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,1024,0.22327680587768556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,1024,0.2869845390319824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,1024,0.286737060546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,768,0.23223573366800943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,768,0.17446293830871581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,512,0.12689066727956136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,512,0.17210666338602704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,256,0.10518293380737305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,256,0.10610986550649007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,512,0.21171627044677735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,128,0.09146347045898437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,128,0.09959039688110352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,256,0.15768319765726727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,64,0.08415893713633218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,32,0.08334186871846518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,64,0.09861120382944742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,65536,32,0.09470933278401693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,1536,0.33147306442260743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,768,0.2472895940144857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,65536,51200,14.64931640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,65536,128,0.1418336073557536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,16384,3.84813232421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,16384,1.9666250864664714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,51200,5.948423258463541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,16384,2.3149131774902343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,65536,7.540483093261718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,51200,7.100527954101563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,12288,1.4578943888346354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,12288,2.741088104248047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,10240,1.183020782470703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,12288,1.7358314514160156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,10240,1.431668217976888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,10240,2.2468416849772135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,65536,11.1768310546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,8192,0.9521504084269206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,51200,11.030447387695313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,8192,1.770081075032552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,7168,0.8550602595011393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,7168,1.632364781697591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,6144,0.7053066889444988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,7168,0.9641461054484048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,6144,0.8589205423990884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,6144,1.3199946085611978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,5120,1.2284341176350912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,8192,1.170130157470703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,5120,0.6230805079142253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,4096,0.486680539449056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,5120,0.6794314702351888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,3584,0.4173429489135742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,4096,0.986084238688151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,3584,0.5514890670776367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,3584,0.7970997492472331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,3072,0.4050463994344075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,3072,0.691485850016276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,3072,0.4226250648498535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,2560,0.3558719952901205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,2560,0.6488597234090169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,2560,0.4206122716267903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,2048,0.45150079727172854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,2048,0.2898698806762695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,2048,0.3282645225524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,1536,0.2388981342315674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,1024,0.2671744028727213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,65536,15.831241861979166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,1024,0.169594669342041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,1536,0.3775967915852865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,1024,0.21358720461527506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,1536,0.294593079884847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,768,0.15221439997355143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,512,0.1361728032430013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,512,0.12981546719868978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,256,0.09380266666412354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,4096,0.6762933095296224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,128,0.07226026852925618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,256,0.08781867027282715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,256,0.13503146171569824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,128,0.08063786824544271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,64,0.06719146569569906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,768,0.19896853764851888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,128,0.11877226829528809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,32,0.06679680347442626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,64,0.08109227021535238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,51200,32,0.08017280101776122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,51200,768,0.19849600791931152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,51200,2.0189163208007814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,65536,2.6951627095540363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,65536,3.0533462524414063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,16384,1.1114507039388022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,51200,2.4381642659505207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,51200,3.4553888956705725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,16384,0.6296106974283855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,12288,0.4726709365844727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,12288,0.6020810445149739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,16384,0.7160202662150066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,10240,0.4141663869222005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,12288,0.8097440083821615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,10240,0.6705610911051433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,65536,4.506477864583333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,10240,0.43996585210164385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,8192,0.35303573608398436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,7168,0.30194454193115233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,7168,0.4892064094543457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,8192,0.6066986719767253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,6144,0.40398079554239913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,7168,0.32877972920735676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,6144,0.27457494735717775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,5120,0.23925654093424478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,5120,0.34392213821411133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,5120,0.24263787269592285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,4096,0.26265066464742026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,4096,0.18778346379597982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,4096,0.20010986328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,3584,0.24519039789835612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,3584,0.1682037353515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,3584,0.17263253529866535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,3072,0.20857599576314292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,3072,0.14571946461995441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,3072,0.15788480440775554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,2560,0.12373226483662922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,2560,0.13649387359619142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,2048,0.14120640754699706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,2048,0.10682453314463299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,2048,0.11233812967936199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,1536,0.11073280175526937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,1536,0.08746559619903564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,1024,0.07622400124867758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,1536,0.09616639614105224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,1024,0.06889066696166993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,8192,0.3633482615152995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,768,0.058964268366495765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,768,0.06936213175455729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,512,0.04499306678771973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,512,0.0494976003964742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,512,0.05532266696294149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,256,0.030366933345794676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,128,0.025949867566426595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,256,0.04072320063908895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,256,0.04660693407058716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,51200,512,0.14473919868469237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,6144,0.26013333002726235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,64,0.024711465835571288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,128,0.03838613430658976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,32,0.025135999917984007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,32,0.038412801424662274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,16384,64,0.0412831981976827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,2560,0.1719381332397461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,16384,768,0.06135786771774292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,1024,0.07665279706319174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,16384,128,0.041554133097330734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,16384,0.7944458643595378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,16384,0.48860692977905273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,65536,2.028375498453776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,51200,1.5180629730224608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,12288,0.5867104212443034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,65536,2.3508544921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,16384,0.5877066930135091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,12288,0.38313493728637693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,51200,2.657640584309896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,51200,1.6778069814046224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,12288,0.40383361180623367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,10240,0.381388791402181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,10240,0.5224362691243489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,10240,0.35923732121785484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,8192,0.3805962562561035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,8192,0.27186559041341146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,65536,3.420905558268229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,8192,0.31353813807169595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,7168,0.35280427932739256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,7168,0.2682453473409017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,7168,0.2530773321787516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,6144,0.29951146443684895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,6144,0.2110677401224772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,6144,0.20939839680989586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,5120,0.18415892918904622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,4096,0.21781439781188966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,5120,0.2600501378377279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,4096,0.14725653330485028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,3584,0.1738965352376302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,3584,0.1339765389760335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,3072,0.11705919901529949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,3584,0.1411754608154297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,3072,0.15493119557698568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,3072,0.11585493087768554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,2560,0.13016640345255534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,2048,0.10489706993103028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,2560,0.09840853214263916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,2048,0.08526826699574788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,2048,0.08907306989034017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,1536,0.08181440035502116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,1536,0.06839893658955892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,1536,0.07299306392669677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,1024,0.059937067826588954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,1024,0.05550186634063721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,1024,0.05593066612879435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,768,0.04713279803593953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,768,0.04826133251190186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,768,0.050460799535115564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,512,0.04182826677958171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,256,0.025667200485865276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,256,0.03474239905675252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,128,0.019947733481725058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,128,0.031240532795588177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,128,0.03239679932594299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,64,0.01772800087928772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,64,0.03149973352750142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,32,0.017948800325393678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,5120,0.18037226994832356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,32,0.031292800108591715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,4096,0.15011626879374187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,2560,0.10318186283111572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,12288,512,0.03602773348490397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,12288,256,0.03489813407262166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,65536,1.790213394165039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,51200,1.3664362589518229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,16384,0.6394858678181966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,16384,0.4523018519083659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,51200,2.326252746582031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,65536,2.933262888590495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,12288,0.3358506520589193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,12288,0.4989002545674642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,10240,0.41576000849405925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,10240,0.2808863957722982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,8192,0.2338698705037435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,8192,0.3480746587117513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,7168,0.30380481084187827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,7168,0.20592640240987142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,65536,2.3391156514485676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,6144,0.29164800643920896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,16384,0.5220394770304362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,12288,512,0.04078720013300578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,51200,1.843008041381836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,6144,0.197926394144694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,5120,0.209168004989624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,12288,0.39690879185994465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,5120,0.16910826365152995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,4096,0.1696202596028646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,4096,0.12554986476898194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,3584,0.15032107035319012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,10240,0.36169919967651365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,3584,0.11568319797515869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,3072,0.13112320105234782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,3072,0.10183786551157634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,2560,0.11189653078715008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,8192,0.2711082776387533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,2560,0.08904853661855062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,2048,0.09186773300170899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,2048,0.07544639905293783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,7168,0.24886080423990883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,1536,0.06970986525217691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,1536,0.061027201016743984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,6144,0.20058347384134928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,1024,0.04989546537399292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,5120,0.16886720657348633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,1024,0.049108266830444336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,768,0.04053013324737549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,768,0.04365013440450032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,4096,0.14887146949768065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,3584,0.12313066323598225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,512,0.03160533308982849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,3072,0.10298773447672527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,512,0.03785813252131144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,256,0.022407466173172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,256,0.03158079981803894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,128,0.017132800817489625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,128,0.028407466411590577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,2560,0.09446506500244141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,64,0.014726400375366211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,2048,0.07815679709116617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,64,0.0283680001894633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,10240,32,0.014899200201034546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,10240,32,0.02799573342005412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,1536,0.05845973491668701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,1024,0.04813333352406819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,65536,1.4090347290039062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,65536,2.312322235107422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,768,0.03896106481552124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,256,0.02719680070877075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,512,0.03152959942817688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,10240,128,0.022881066799163817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,16384,0.49595734278361003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,16384,0.35934292475382484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,16384,0.35944105784098307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,51200,1.1143850962320963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,12288,0.4095904032389323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,51200,1.1699935913085937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,10240,0.3199541409810384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,65536,1.5084512074788412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,51200,1.67345708211263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,10240,0.24461973508199059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,8192,0.19359679222106935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,8192,0.2276906649271647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,8192,0.3145941416422526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,7168,0.2619200070699056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,7168,0.18350720405578613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,6144,0.16093653043111164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,6144,0.23719573020935059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,5120,0.1668597380320231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,5120,0.13259946505228679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,5120,0.135915740331014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,4096,0.13109546502431232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,4096,0.10895466804504395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,4096,0.10705493291219075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,3584,0.09805866877237955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,3584,0.0924618641535441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,3072,0.10225066343943279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,12288,0.2795680046081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,3072,0.08695360024770102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,3072,0.08134293556213379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,2560,0.07487359841664633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,10240,0.24200000762939453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,2048,0.07015039920806884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,2048,0.06329919894536337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,2048,0.05994240045547485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,1536,0.05515199899673462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,1536,0.053763198852539065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,1536,0.048648532231648764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,1024,0.04029546578725179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,1024,0.042906665802001955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,1024,0.039854931831359866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,6144,0.16064853668212892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,768,0.03857066631317139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,512,0.025197867552439374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,512,0.03344426552454631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,3584,0.11608106295267742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,512,0.02954559922218323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,256,0.01817173361778259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,256,0.029066665967305498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,256,0.02480319937070211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,128,0.014206932981808982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,128,0.026603732506434125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,128,0.022925867636998495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,64,0.011995733777681986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,32,0.012454400459925335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,64,0.02804693380991618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,8192,32,0.027245867252349853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,2560,0.08576213518778483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,2560,0.0696618636449178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,8192,768,0.032755200068155924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,768,0.03483626842498779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,65536,1.310706075032552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,51200,1.4717386881510417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,51200,1.0351637522379558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,65536,2.052610143025716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,16384,0.47583999633789065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,12288,0.2693589210510254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,16384,0.3548682530721029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,12288,0.37446721394856775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,10240,0.2940352121988932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,10240,0.23215999603271484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,8192,0.2298901398976644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,8192,0.1805066744486491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,12288,0.27825174331665037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,7168,0.20740267435709633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,7168,0.16110827128092448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,8192,7168,0.16655786832173664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,6144,0.17672746976216633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,6144,0.13195626735687255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,12288,0.27281068166097006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,5120,0.14522453943888347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,5120,0.12511146863301595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,16384,0.370307191212972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,4096,0.12018667062123616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,4096,0.10512213706970215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,65536,1.6049823760986328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,10240,0.23439466158548988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,3584,0.11358826955159504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,3584,0.09420053164164224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,3072,0.0915018637975057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,3072,0.08148586750030518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,51200,1.1883008321126303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,8192,0.18513387044270832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,2560,0.07596586545308431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,6144,0.1413376013437907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,2560,0.07034986813863119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,2048,0.06815573374430338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,2048,0.060405333836873375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,7168,0.16892053286234537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,1536,0.05451519886652628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,1536,0.049617067972819014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,1024,0.0375434676806132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,1024,0.03935679992039998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,3072,0.07436693509419759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,768,0.02940906683603923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,768,0.03508053223292033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,5120,0.11861653327941894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,4096,0.09927999973297119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,512,0.022498132785161336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,512,0.03135786652565002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,256,0.016659200191497803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,256,0.028091732660929365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,1536,0.04309653441111247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,128,0.013931733369827271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,128,0.026014933983484905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,3584,0.08615360260009766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,64,0.01160426636536916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,64,0.02518826723098755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,7168,32,0.01209920048713684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,7168,32,0.025514666239420575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,2560,0.06538240114847818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,2048,0.05458879868189494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,65536,1.59901123046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,65536,1.1456874847412108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,51200,0.9113706588745117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,51200,1.2498570760091146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,1024,0.03535146713256836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,16384,0.2936511993408203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,16384,0.40529279708862304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,256,0.01893333395322164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,12288,0.2981173197428385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,12288,0.22350187301635743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,10240,0.23743893305460612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,512,0.023528534173965453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,10240,0.19850559234619142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,8192,0.1918527921040853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,128,0.016193067034085594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,7168,768,0.02804479996363322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,8192,0.16404372851053875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,7168,0.16812480290730794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,7168,0.14371412595113117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,6144,0.15232213338216144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,6144,0.11907093524932862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,65536,1.3557664235432942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,5120,0.13384853998819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,5120,0.1128970702489217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,16384,0.3152607917785645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,4096,0.09799573421478272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,51200,1.01909548441569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,4096,0.08574933211008708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,3584,0.08681813081105551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,12288,0.2572458744049072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,3584,0.07746026515960694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,3072,0.07516372998555501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,10240,0.22990934054056802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,3072,0.06916906833648681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,2560,0.06364800135294596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,2560,0.06125760078430176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,8192,0.16796053250630696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,2048,0.053157333532969156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,2048,0.05267519950866699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,2048,0.047891199588775635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,7168,0.14460372924804688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,1536,0.041116801897684734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,1536,0.04271466732025146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,1024,0.02881173292795817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,1024,0.034281599521636966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,6144,0.12221226692199708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,768,0.023729066054026283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,768,0.030473599831263225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,768,0.024618667364120484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,512,0.01836479902267456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,512,0.02725013295809428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,5120,0.1040181318918864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,256,0.013143466909726462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,256,0.022794665892918904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,4096,0.08590506712595622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,128,0.00988159974416097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,128,0.02109866738319397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,3584,0.07482133706410726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,3072,0.06265706618626912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,64,0.008390399813652038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,64,0.02123946746190389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,6144,32,0.008657067020734151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,6144,32,0.021192532777786256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,2560,0.058234667778015135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,1536,0.03755626678466797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,1024,0.03205759922663371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,65536,1.017075220743815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,65536,1.403762181599935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,65536,1.1069525400797526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,512,0.02044693430264791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,51200,1.0666848500569661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,16384,0.29520107905069987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,16384,0.36884053548177087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,51200,0.8134058634440103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,12288,0.27041600545247396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,51200,0.8821919759114584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,16384,0.2982080141703288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,12288,0.21384107271830238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,256,0.01607039968172709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,8192,0.16326613426208497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,10240,0.21726080576578774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,10240,0.19071040153503419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,8192,0.13695359230041504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,7168,0.15437760353088378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,8192,0.14767146110534668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,7168,0.13076799710591633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,6144,0.10824106534322102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,5120,0.1042357365290324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,5120,0.09291840394337972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,6144,128,0.013989333311716715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,5120,0.09413546721140544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,4096,0.08524160385131836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,4096,0.07877866427103677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,3584,0.07423040072123209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,3584,0.07089493274688721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,3584,0.06820693016052246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,10240,0.18675732612609863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,3072,0.0645578662554423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,3072,0.06440853277842204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,3072,0.05713066657384237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,2560,0.055719467004140225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,2560,0.05528639952341715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,2048,0.04514986673990885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,6144,0.13266773223876954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,2048,0.047074135144551596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,1536,0.03523840109507243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,1536,0.038924801349639895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,1536,0.03431253433227539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,1024,0.02574719985326131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,1024,0.031753599643707275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,768,0.021038933595021566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,768,0.028508800268173217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,768,0.022797866662343343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,512,0.016402133305867515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,12288,0.21382613182067872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,512,0.024948267141977946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,512,0.01912533243497213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,256,0.02367253303527832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,256,0.015565866231918335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,128,0.010713600118954976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,7168,0.13481280008951824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,128,0.022360533475875854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,6144,0.11068586508433025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,64,0.009800533453623455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,32,0.010295466581980387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,64,0.02260266741116842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,5120,32,0.022677334149678548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,4096,0.08000319798787435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,5120,256,0.01139520009358724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,65536,0.9070218404134115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,65536,1.1518709818522135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,2560,0.05140053431193033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,2048,0.04432426691055298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,65536,0.816806411743164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,16384,0.2559541384379069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,12288,0.19358080228169758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,51200,0.7191359837849934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,51200,0.8624149322509765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,16384,0.2203754742940267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,51200,0.6819189071655274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,12288,0.17775786717732747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,10240,0.1609674612681071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,10240,0.18448853492736816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,1024,0.02802346746126811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,8192,0.13097386360168456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,10240,0.15690773328145344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,8192,0.12676373322804768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,7168,0.11580586433410645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,7168,0.10797866980234783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,7168,0.10646826426188152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,6144,0.10101546446482342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,5120,0.09271039962768554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,6144,0.08242666721343994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,6144,0.09321706295013428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,5120,0.08465813000996908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,4096,0.06754559675852458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,5120,0.07467839717864991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,4096,0.06823466618855795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,4096,0.05936959981918335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,3584,0.0598751982053121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,3584,0.051249066988627114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,3584,0.06718293031056723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,3072,0.05515413284301758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,2560,0.04760426680246989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,3072,0.04514559904734294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,2560,0.04795200030008952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,2048,0.03572373390197754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,2560,0.04165866772333781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,2048,0.04148586591084798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,1536,0.02947946588198344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,5120,128,0.013566933075586953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,2048,0.034456535180409746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,1536,0.035412267843882246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,1024,0.020504534244537354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,1536,0.028675200541814168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,768,0.016901334126790367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,1024,0.029105067253112793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,16384,0.23432319959004722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,768,0.02581760088602702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,512,0.013150933384895324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,512,0.017498666048049928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,512,0.022974934180577597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,256,0.009458133578300476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,256,0.020444800456364952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,256,0.014434132973353067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,128,0.007528533538182576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,128,0.019048533837000527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,64,0.006562133133411407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,64,0.019059199094772338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,32,0.006888533135255177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,4096,32,0.019102933009465535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,4096,3072,0.052840534845987955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,65536,0.959283192952474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,65536,0.8304821650187174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,65536,0.920034154256185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,51200,0.7329706827799479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,16384,0.2320181369781494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,51200,0.6737887700398763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,51200,0.6941706975301106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,16384,0.22040747006734213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,12288,0.17863787015279134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,12288,0.167795197168986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,10240,0.14657386144002277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,10240,0.13981226285298665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,128,0.0135754664738973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,8192,0.11451626618703206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,8192,0.10927146275838215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,8192,0.10553173224131267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,7168,0.10134080251057942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,7168,0.09949226379394531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,16384,0.20663679440816246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,12288,0.15400853157043456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,6144,0.08849279880523682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,12288,0.15675519307454427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,6144,0.08818666934967041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,5120,0.07505066394805908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,5120,0.07683839797973632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,4096,0.06106773217519125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,4096,0.06357973416646322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,10240,0.13528107007344564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,4096,0.05600746472676595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,3584,0.05418773492177328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,3584,0.05768106778462728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,3072,0.046199464797973634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,3072,0.05033173163731893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,768,0.020409599939982096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,2560,0.038106667995452884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,2560,0.04375786781311035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,8192,0.10642879803975422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,2048,0.0316266675790151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,2048,0.03760213454564412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,7168,0.09362133344014487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,1536,0.024755199750264488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,1536,0.031947733958562215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,6144,0.08017173608144125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,5120,0.06772800286610922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,1024,0.018309332927068076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,1024,0.026314665873845417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,768,0.015105066696802774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,768,0.024846933285395303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,3584,0.050969600677490234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,512,0.011538133025169373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,512,0.02264639933904012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,3072,0.043848534425099686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,256,0.008634666601816814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,256,0.020768000682195028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,256,0.01179200013478597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,128,0.007291733225186665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,128,0.01920106609662374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,128,0.01058240036169688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,2560,0.03902080059051514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,64,0.006363733112812043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3584,32,0.006697600086530049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,32,0.01946453253428141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,1024,0.021284266312917074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,2048,0.03219946622848511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,65536,0.744485346476237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,4096,1024,0.02296533385912577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,1536,0.025887999931971235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,65536,0.7436319986979167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,16384,0.18633386294047039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3584,64,0.019512534141540527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,51200,0.6333269119262696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,16384,0.1987317403157552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,12288,0.15693012873331708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,51200,0.5857173283894856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,12288,0.15107733408610027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,10240,0.1180191993713379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,10240,0.1269002676010132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,8192,0.09544533093770345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,768,0.0173418660958608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,8192,0.10152640342712402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3584,512,0.014589866995811463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,7168,0.08555306593577067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,7168,0.09241066773732504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,6144,0.0726154645284017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,6144,0.08018346627553305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,12288,0.13807253837585448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,5120,0.06152106523513794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,5120,0.06995413303375245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,16384,0.18004159927368163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,4096,0.04918613433837891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,65536,0.7286303838094075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,4096,0.05859946807225545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,3584,0.0434559981028239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,3584,0.05260159969329834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,10240,0.11836480299631755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,3072,0.038179198900858566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,51200,0.5753834406534831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,3072,0.04565440018971761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,2560,0.032414933045705155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,2560,0.03988693157831828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,7168,0.08472747008005778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,2048,0.02771199941635132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,2048,0.03560426632563273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,8192,0.09701440334320069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,1536,0.021569067239761354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,1536,0.030246400833129884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,6144,0.07033600012461344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,1024,0.0163263996442159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,1024,0.025497599442799883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,3072,0.03970986604690552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,768,0.013589333494504294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,768,0.024596265951792397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,4096,0.05111253261566162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,512,0.010514133175214132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,512,0.02141439914703369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,5120,0.061912532647450766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,256,0.007946666578451793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,256,0.019399466117223103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,1536,0.024098134040832518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,128,0.006594133377075195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,128,0.01806933283805847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,3584,0.045823999245961505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,64,0.005869866907596588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,64,0.01856213410695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,3072,32,0.0059456000725428265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,3072,32,0.01798186699549357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,2560,0.03557866811752319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,65536,0.6330005645751953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,65536,0.6925578435262044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,2048,0.029809067646662395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,65536,0.7333546956380208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,51200,0.5065749486287434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,51200,0.5530613581339519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,16384,0.17096959749857585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,1024,0.019976532459259032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,16384,0.18700693448384603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,51200,0.558946164449056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,12288,0.1223370631535848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,12288,0.14369813601175946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,10240,0.1026911973953247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,10240,0.12046506404876708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,8192,0.08449172973632812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,8192,0.09723412990570068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,256,0.012012799580891926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,7168,0.07357227007548015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,128,0.010702932874361675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,7168,0.086080002784729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,7168,0.08246933619181315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,6144,0.06379626592000326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,6144,0.07545173168182373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,512,0.014582399527231851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,5120,0.05387200117111206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,5120,0.0655679980913798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,4096,0.043849599361419675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,4096,0.054474667708079016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,4096,0.04941759904225667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,3584,0.03901546796162923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,3072,768,0.01644159952799479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,3584,0.048342398802439374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,3584,0.04183893203735352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,3072,0.0339626669883728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,3072,0.04269653161366781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,3072,0.03585600058237712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,2560,0.029075199365615846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,2560,0.03746986786524455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,2048,0.02378986676534017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,2048,0.0328821341196696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,2048,0.026447999477386474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,1536,0.01930346687634786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,1536,0.028654932975769043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,1536,0.021086933215459187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,1024,0.014615466197331747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,1024,0.023996800184249878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,1024,0.016889599959055583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,768,0.012129066387812297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,768,0.022347732384999593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,768,0.014436266819636025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,512,0.009611733754475911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,12288,0.13479894002278645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,512,0.020374399423599244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,512,0.012107732892036437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,256,0.007249066730340321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,256,0.018550399939219156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,256,0.010179199775060018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,128,0.0061034664511680605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,128,0.017614932854970296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,128,0.00920960009098053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,64,0.005347200234731038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,64,0.01742080052693685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2560,32,0.005341866612434387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2560,32,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,65536,0.5050549189249675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,16384,0.17201493581136068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,10240,0.11786453723907471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,65536,0.6487424214680989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,51200,0.39829012552897136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,51200,0.4998517354329427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,16384,0.1287893295288086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,6144,0.06842026710510254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,16384,0.16745386123657227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,51200,0.5287530581156413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,12288,0.09740800062815348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,12288,0.13816213607788086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,10240,0.08815146287282308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,10240,0.10805119673411052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,8192,0.06724373499552408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,8192,0.08778026898701986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,8192,0.0932032028834025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,7168,0.059780267874399814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,7168,0.07891519864400229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,2560,0.031779199838638306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,6144,0.05038719971974691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,6144,0.06861973603566487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,65536,0.6739498774210613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,5120,0.04377813339233398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,5120,0.06141440073649088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,16384,0.16893866856892903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,4096,0.035267198085784913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,12288,0.12635626792907714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2560,5120,0.057963732878367105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,4096,0.04756480058034261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,3584,0.03097599943478902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,3584,0.042634665966033936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,3584,0.03627626498540242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,3072,0.02694293260574341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,3072,0.0378549337387085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,2560,0.022909865776697794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,10240,0.10916586716969807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,2560,0.03373653491338094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,2560,0.028205867608388262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,2048,0.01917120019594828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,2048,0.030687999725341798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,8192,0.08077973524729411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,1536,0.01588373382886251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,1536,0.026449066400527955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,1024,0.011492266257603963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,7168,0.071998929977417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,1024,0.02254613240559896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,768,0.00974826713403066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,768,0.02213546633720398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,6144,0.061741864681243895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,512,0.007947733501593272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,512,0.019644800821940103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,512,0.010915199915568035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,256,0.006190933287143707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,5120,0.05055893262227377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,4096,0.04292053381601969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,128,0.005310933291912079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,128,0.01697280009587606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,128,0.008356266220410665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,64,0.004817066589991251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,3072,0.03129706581433614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,2048,32,0.004961066444714864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,64,0.017148800690968833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,32,0.017172267039616905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,2048,0.022820266087849934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,65536,0.3781418800354004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,1536,0.018371200561523436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,1024,0.015169066190719605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,768,0.012730666995048523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,2048,256,0.01826453407605489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,65536,0.577397346496582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,51200,0.3122389475504557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,16384,0.11705280145009359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,12288,0.08869973023732504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,16384,0.1597226619720459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,51200,0.47501548131306964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,12288,0.12432746887207032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,10240,0.07517759799957276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,10240,0.10490453243255615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,8192,0.05875946680704752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,8192,0.08554879824320474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,7168,0.05297386646270752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,7168,0.07562026977539063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,2048,256,0.009194667140642803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,6144,0.04662826855977376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,6144,0.0656063993771871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,12288,0.11277973651885986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,16384,0.15398933092753092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,65536,0.611842155456543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,5120,0.040504535039265946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,5120,0.05569599866867066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,10240,0.09722879727681479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,4096,0.03235519925753276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,4096,0.044726399580637614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,3584,0.0284768005212148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,3584,0.04024853308995564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,51200,0.47710294723510743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,3072,0.025415466229120894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,7168,0.06585386594136557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,3072,0.03690559864044189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,2560,0.021740800142288207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,2560,0.03270080089569092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,8192,0.07237760225931802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,2048,0.017883733908335368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,2048,0.028655999898910524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,1536,0.014538666605949402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,1536,0.024959999322891235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,6144,0.05425066550572714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,1024,0.010818133751551311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,1024,0.02140586574872335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,3072,0.02883946696917216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,768,0.009032533566157023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,768,0.020150399208068846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,4096,0.03813013235727946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,512,0.006664533416430156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,512,0.018905599912007652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,2560,0.025727999210357667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,256,0.005398400127887726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,256,0.017388800779978432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,3584,0.033529599507649735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,128,0.004609066744645437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,128,0.01676586667696635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,5120,0.04745386838912964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,1536,0.01681386629740397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,64,0.004215466479460398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1536,32,0.004465066889921824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,64,0.01660160024960836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1536,32,0.016450132926305136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,65536,0.26430932680765784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,65536,0.5153226534525553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,51200,0.21309866905212402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,51200,0.4072330792744954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,2048,0.021167999505996703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,51200,0.327837880452474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,16384,0.08311039606730143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,16384,0.14602239926656085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,16384,0.11163840293884278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,12288,0.06160533428192139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,12288,0.11446719964345295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,10240,0.057061334451039634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,1024,0.01360319952170054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,10240,0.09672746658325196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,8192,0.04395093520482381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,10240,0.07064747015635173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,8192,0.0573802669843038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,8192,0.07226026852925618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,7168,0.04240533510843913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,7168,0.06369173526763916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,6144,0.03629973332087199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,6144,0.054780801137288414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,6144,0.044468267758687334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,5120,0.02983786662419637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,5120,0.04536853233973186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,65536,0.42287254333496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,4096,0.018652800718943277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,4096,0.037622400124867755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,4096,0.03152959942817688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,3584,0.017859200636545815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,3584,0.03434346516927083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,128,0.007653333246707916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,256,0.00848640004793803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,3072,0.01583253343900045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,512,0.010216533144315084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,2560,0.014292266964912415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,3072,0.031155200799306233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,2560,0.02834133307139079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,2560,0.02073600093523661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,2048,0.011026133100191753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,2048,0.025515733162562053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,1536,0.009217066566149394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,1536,0.022232532501220703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,12288,0.08434666792551676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,1024,0.007289599875609081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1536,768,0.011870933572451274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,1024,0.02007360061009725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,768,0.006367999811967213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,768,0.01906026601791382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,7168,0.05070399840672811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,512,0.0054506664474805195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,512,0.017938133080800375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,512,0.008599467078844706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,256,0.004530133306980133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,5120,0.03749653498331706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,256,0.016692266861597697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,128,0.004009599983692169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,128,0.016235733032226564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,3584,0.027677865823109944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,3072,0.023766400416692098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,64,0.0036608000596364343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,64,0.015946666399637856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,1024,32,0.0038602667550245917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,2048,0.017576533555984496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,1024,32,0.015863466262817382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,1536,0.014378666877746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,65536,0.20306879679361978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,51200,0.16160640716552735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,1024,0.01129919985930125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,65536,0.48273601531982424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,65536,0.3917695999145508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,768,0.009845333298047383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,16384,0.0558837334314982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,12288,0.0426144003868103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,16384,0.1012885332107544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,16384,0.13088213602701823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,51200,0.3850474675496419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,10240,0.03642559846242269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,12288,0.11154666741689045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,8192,0.029406932989756267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,10240,0.0949184020360311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,8192,0.07434773445129395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,8192,0.0525322675704956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,7168,0.03506666819254557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,7168,0.064192001024882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,256,0.007160533467928569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,6144,0.030696533123652142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,6144,0.055021866162618005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,1024,128,0.0064640000462532045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,5120,0.025805866718292235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,5120,0.04495999813079834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,4096,0.02204266587893168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,4096,0.037511467933654785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,3584,0.01882773240407308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,3584,0.03434240023295085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,3584,0.02503146727879842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,3072,0.016938666502634682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,3072,0.031717334191004434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,51200,0.3039039929707845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,12288,0.07777173519134521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,2560,0.014197333653767904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,10240,0.06526293357213339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,2560,0.02861013412475586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,2048,0.011678933103879293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,2048,0.025123200813929242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,1536,0.009404800335566203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,1536,0.02230613430341085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,7168,0.04729386568069458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,1024,0.0070720002055168155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,1536,0.013238400220870972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,6144,0.04096853335698446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,1024,0.019821866353352865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,768,0.006177066763242086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,768,0.01895786722501119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,512,0.005300266544024149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,5120,0.03425813515981038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,512,0.018042665719985963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,256,0.004507733384768168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,256,0.01667520006497701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,4096,0.027872000137964887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,128,0.0040224000811576845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,128,0.015613866845766702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,128,0.006129066646099091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,64,0.0037941334148248037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,64,0.015646933515866598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,768,32,0.003988266736268997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,768,32,0.015719466408093772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,65536,0.14197440147399903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,3072,0.021924267212549843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,65536,0.44927145640055344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,51200,0.11368746757507324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,51200,0.35157759984334314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,2560,0.018780799706776936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,2048,0.015966932972272238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,16384,0.04532373348871867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,16384,0.12643626530965169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,12288,0.03656426668167114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,1024,0.010598400235176086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,768,0.009139200051625569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,512,0.008098133405049642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,10240,0.08580586910247803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,768,256,0.006638933221499126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,8192,0.024539732933044435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,8192,0.06674026648203532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,12288,0.09795839786529541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,7168,0.0209824005762736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,10240,0.031548800071080525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,7168,0.05753920078277588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,6144,0.01826453407605489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,6144,0.048492801189422605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,65536,0.3611861228942871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,5120,0.019032533963521323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,5120,0.040329599380493165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,12288,0.0719381332397461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,4096,0.015035733580589294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,4096,0.033930667241414386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,51200,0.28639361063639324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,16384,0.09216427008310954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,3584,0.013382400075594584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,3584,0.030462932586669923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,10240,0.06025919914245605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,3072,0.012729600071907043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,3072,0.028413865963617963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,6144,0.037375998497009275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,8192,0.04844906727472941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,2560,0.010507733623186747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,2560,0.02566186587015788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,2048,0.008729599912961324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,2048,0.024238934119542442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,7168,0.04233813285827637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,1536,0.007603199779987335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,1536,0.020980266729990642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,1024,0.005740800003210703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,1024,0.018410666783650716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,3072,0.019800533850987755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,768,0.005189333359400431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,4096,0.025523199637730913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,768,0.018258132537206016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,512,0.0043488000830014546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,512,0.016973867019017538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,5120,0.030983465909957885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,256,0.003868799904982249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,256,0.01611840029557546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,3584,0.022360533475875854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,128,0.0034976000587145484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,128,0.015549866358439126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,1536,0.012040533622105916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,64,0.003291733314593633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,2560,0.017046399911244712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,512,32,0.0033802665770053864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,64,0.015266133348147073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,512,32,0.015412267049153647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,65536,0.09525653521219889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,2048,0.014434132973353067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,65536,0.4249418576558431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,51200,0.07684480349222819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,51200,0.337445322672526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,16384,0.030830933650334673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,16384,0.1195530652999878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,1024,0.009176533420880635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,12288,0.02502400080362956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,12288,0.09199360211690268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,256,0.006072533130645752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,10240,0.021118932962417604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,10240,0.07654399871826172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,512,0.007422933479150136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,8192,0.016673066218694053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,8192,0.0595583995183309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,128,0.00565226674079895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,7168,0.0150026669104894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,512,768,0.008283733328183492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,7168,0.05105599959691366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,6144,0.013132799665133157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,6144,0.04285759925842285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,65536,0.3263776143391927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,5120,0.011720533172289532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,5120,0.036142933368682864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,51200,0.26230506896972655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,4096,0.010024533669153849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,4096,0.031251200040181476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,16384,0.08296746412913004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,3584,0.0092031995455424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,3584,0.028516266743342084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,12288,0.06481173435846964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,3072,0.008353066444396973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,3072,0.025849600632985432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,10240,0.05551040172576904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,2560,0.007826133569081625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,2560,0.024117332696914674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,6144,0.03369173208872477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,7168,0.039163732528686525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,8192,0.042982399463653564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,2048,0.006187733511130015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,1536,0.005538133283456167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,2048,0.022290132443110146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,1536,0.02061226765314738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,1024,0.004774400095144907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,1024,0.018385066588719686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,5120,0.028524800141652422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,768,0.004267733295758565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,768,0.017591466506322227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,4096,0.023448532819747923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,512,0.0038090666135152185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,512,0.016871466239293417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,3072,0.018152532974878947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,256,0.003479466587305069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,3584,0.02053119937578837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,256,0.015759999553362526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,128,0.0032266666491826378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,128,0.014862933754920959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,2560,0.015970133741696677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,64,0.0030517332255840302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,64,0.015451733271280924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,256,32,0.003124266614516576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,256,32,0.015246933698654175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,65536,0.08300800323486328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,1536,0.01130239963531494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,1024,0.00897813340028127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,51200,0.06713173389434815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,65536,0.26295359929402673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,51200,0.3300661404927572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,2048,0.013496533036231995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,65536,0.4181130727132161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,16384,0.026229333877563477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,768,0.007896533111731212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,12288,0.01917440096537272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,16384,0.07053226629892985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,16384,0.11537280082702636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,12288,0.05443520148595175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,10240,0.01362773378690084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,12288,0.09059092998504639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,256,0.005959466596444448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,8192,0.011623467008272808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,10240,0.07469759782155355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,7168,0.011000532905260723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,8192,0.03710613250732422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,8192,0.057450668017069495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,7168,0.03312533299128215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,6144,0.04190719922383626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,6144,0.011808000008265178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,6144,0.028172800938288372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,5120,0.010392533739407857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,5120,0.03495039939880371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,4096,0.008713600039482117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,5120,0.024245333671569825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,4096,0.030572799841562907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,3584,0.009498666723569233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,3584,0.027026132742563887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,128,0.005365333457787832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,3072,0.009197866916656494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,3072,0.026097067197163898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,2560,0.006754133105278015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,2560,0.02369173367818197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,2560,0.014435199896494546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,2048,0.006161066889762879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,7168,0.050635735193888344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,2048,0.02332906723022461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,2048,0.012626133362452188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,1536,0.0052928000688552855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,4096,0.020244266589482626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,1536,0.02040426731109619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,1024,0.004648533463478088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,1024,0.018863999843597413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,51200,0.20670080184936523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,256,512,0.007053866485754649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,768,0.017373865842819212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,512,0.0036992001036802924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,768,0.007901866734027863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,512,0.007257600128650665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,512,0.016506666938463845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,256,0.003369600077470144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,256,0.01581653356552124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,256,0.0061749334136645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,128,0.015030399958292643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,10240,0.04531306823094686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,64,0.002930133293072383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,64,0.01529706617196401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,32,0.0029834667841593427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,128,32,0.014883200327555338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,768,0.004165333261092504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,65536,0.07922240098317465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,51200,0.06279573440551758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,3584,0.01846933364868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,16384,0.023986132939656575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,128,128,0.0031146667897701263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,65536,0.4148586591084798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,12288,0.015565866231918335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,16384,0.11330453554789226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,51200,0.32589546839396155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,10240,0.011549866199493409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,8192,0.009786666433016459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,12288,0.08643733660380046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,7168,0.009194667140642803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,10240,0.0724554697672526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,6144,0.008597333232561748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,8192,0.0547765334447225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,5120,0.008046933511892954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,6144,0.03983253240585327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,7168,0.047637331485748294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,5120,0.033752532800038655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,4096,0.007355733215808869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,3584,0.007657599945863088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,4096,0.028849067290623982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,3584,0.028014934062957762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,2560,0.00639573335647583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,2048,0.005601066847642263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,2560,0.02370880047480265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,1536,0.004982399940490723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,2048,0.022115200757980347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,1024,0.0042357335488001505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,1536,0.020148267348607383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,1536,0.010898133118947346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,768,0.003903999924659729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,1024,0.018221867084503175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,512,0.003588266670703888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,768,0.017859200636545815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,256,0.0032074667513370516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,512,0.016463999946912132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,128,0.002919466545184453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,256,0.01546986699104309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,1024,0.008797867099444072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,64,0.0027850667635599775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,32,0.0027903998891512555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,64,0.015079466501871744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,32,0.014758400122324624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,65536,0.07740159829457602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,3072,0.016378666957219443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,51200,0.061654400825500486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,16384,0.02303253412246704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,64,3072,0.006966400146484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,3072,0.025961599747339886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,16384,0.11263039906819661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,65536,0.4120479901631673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,12288,0.011342933773994446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,51200,0.3254176139831543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,10240,0.01067733367284139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,8192,0.009051733215649923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,7168,0.008520533641179402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,12288,0.08559679985046387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,10240,0.07135679721832275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,8192,0.053481598695119224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,6144,0.00788800021012624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,7168,0.045466665426890054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,5120,0.007242666681607564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,4096,0.006611200173695882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,3584,0.00661653329928716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,6144,0.038040534655253096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,4096,0.02895680069923401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,5120,0.03295039931933085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,3584,0.027365332841873168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,2560,0.0061930666367212926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,3072,0.02553706765174866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,2048,0.005549866457780202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,1536,0.004914133250713349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,2560,0.023676800727844238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,2048,0.02144320011138916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,1024,0.0041461333632469176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,1536,0.019732266664505005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,768,0.003849600007136663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,512,0.003532800078392029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,1024,0.018167465925216675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,768,0.017297067244847617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,256,0.0031338666876157125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,128,0.0030165334542592366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,256,0.015613866845766702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,64,0.0028736000259717304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,128,0.014972800016403198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,32,0.002887466549873352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,64,0.015082666277885437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,32,0.014663466811180114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,64,128,0.015222400426864624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1536,128,128,0.005679999788602194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1536,32,3072,0.006268799801667531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1536,32,512,0.016277333100636802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,16384,1.963112513224284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,16384,3.9828244527180994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,12288,1.4807029724121095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,12288,2.9580960591634113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,16384,2.649735514322917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,12288,2.051745096842448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,10240,1.2957322438557943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,10240,2.5186612447102865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,10240,1.6916053771972657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,8192,1.880071512858073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,8192,0.9995562871297201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,51200,6.539227803548177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,8192,1.2895712534586588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,7168,0.8575306574503581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,7168,1.5452074686686197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,6144,0.7414250691731771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,7168,1.2075125376383462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,6144,1.427341842651367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,6144,1.0197706858317057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,5120,0.7011914571126302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,5120,1.243075180053711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,5120,0.8460330963134766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,3584,0.4491008122762044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,4096,0.9254079818725586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,4096,0.5109610557556152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,4096,0.6995274861653645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,3584,0.8091850916544596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,3584,0.5991071701049805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,3072,0.41243521372477215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,3072,0.5374069213867188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,2560,0.3510730743408203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,3072,0.718349838256836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,2560,0.6143722534179688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,2048,0.30306879679361975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,1536,0.25733653704325354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,2560,0.4601301193237305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,2048,0.47185707092285156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,2048,0.39096320470174156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,1536,0.39821653366088866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,51200,11.836161295572918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,768,0.19977386792500812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,1024,0.26728318532307943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,1536,0.31764052708943685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,768,0.15788052876790365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,1024,0.2794303894042969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,256,0.09968960285186768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,512,0.15445119539896648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,512,0.11935146649678548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,512,0.18056000073750814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,51200,12.092289225260417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,128,0.08219520250956217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,768,0.2334730625152588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,256,0.09566720326741537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,256,0.1599925359090169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,128,0.08657279809316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,64,0.0739039977391561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,65536,32,0.07174293200174967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,32,0.0842357317606608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,64,0.0847658634185791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,65536,128,0.16009173393249512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,65536,1024,0.17856000264485677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,16384,3.190622965494792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,16384,1.6563242594401042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,51200,5.493612670898438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,16384,2.2053738911946614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,65536,6.341351318359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,12288,2.3576649983723956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,12288,1.251317342122396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,12288,1.5978304545084634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,10240,0.9996821085611979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,10240,1.9911253611246746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,8192,0.8112981160481771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,10240,1.3615914662679036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,51200,9.799305216471355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,8192,1.4492778778076172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,7168,0.7455882390340169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,8192,1.0909280141194662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,7168,1.33067626953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,65536,11.582216389973958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,6144,0.6291946411132813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,7168,0.9114901224772135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,51200,9.57141621907552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,6144,0.8819797515869141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,5120,0.9070101420084635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,5120,0.4998517354329427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,4096,0.3973312060038249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,5120,0.671559460957845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,4096,0.7339424133300781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,4096,0.5598655700683594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,65536,12.842695109049478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,3584,0.37099946339925133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,3584,0.6471978505452474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,3072,0.3484437306722005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,6144,1.0685919443766276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,3584,0.476531187693278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,3072,0.4434176127115886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,3072,0.574565315246582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,2560,0.2735797246297201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,2560,0.36899305979410807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,1536,0.1996330738067627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,2048,0.42931413650512695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,1536,0.3112586657206217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,2048,0.30879147847493493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,1536,0.25653012593587243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,1024,0.1484074592590332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,1024,0.19523839950561522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,768,0.11933546861012775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,768,0.17650453249613446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,1024,0.23597332636515297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,512,0.12315626939137776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,512,0.09516586462656657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,256,0.08875733216603597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,768,0.18584532737731935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,512,0.1341610590616862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,128,0.06604373455047607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,64,0.058974933624267575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,256,0.09020480314890543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,128,0.11136426925659179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,51200,256,0.12438826560974121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,32,0.05867840051651001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,64,0.07154133319854736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,32,0.0731498638788859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,2048,0.21948906580607094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,51200,2560,0.5566261291503907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,51200,128,0.07619840304056803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,16384,0.9186997095743814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,16384,0.5453877131144206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,51200,1.8043275197347004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,65536,2.3760426839192705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,16384,0.6940511703491211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,12288,0.7021461486816406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,51200,2.1710474650065104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,12288,0.5136351903279622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,10240,0.3806730588277181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,51200,3.544017028808594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,10240,0.6220949172973633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,8192,0.29580799738566077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,10240,0.45545066197713213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,8192,0.49988800684611007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,7168,0.4495210647583008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,65536,3.9982154846191404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,7168,0.2597930590311686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,12288,0.4580138524373372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,7168,0.31343361536661785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,6144,0.23902079264322915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,6144,0.3928373336791992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,5120,0.291865603129069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,6144,0.28174079259236656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,5120,0.18621546427408855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,5120,0.23171733220418295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,4096,0.2661802609761556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,4096,0.16120212872823078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,3584,0.1402016003926595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,3584,0.2061631997426351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,3584,0.17784959475199383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,3072,0.12503146330515544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,3072,0.1379317283630371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,2560,0.1637066682179769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,2560,0.10797866980234783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,2560,0.11997013092041016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,2048,0.12968213558197023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,2048,0.09256532986958822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,2048,0.10181012948354085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,1536,0.10222613016764323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,8192,0.3785344123840332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,1536,0.07606826623280843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,1024,0.07489066918691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,1024,0.058692268530527746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,1024,0.065993599096934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,768,0.0582751989364624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,768,0.0510538657506307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,512,0.04185173511505127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,512,0.0434058666229248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,512,0.04890880187352498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,256,0.035784534613291424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,256,0.04107840061187744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,128,0.023299199342727662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,4096,0.1834997336069743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,128,0.03431573311487834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,3072,0.18023573557535807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,128,0.03760533332824707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,32,0.021010132630666097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,64,0.035214932759602864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,16384,32,0.035434667269388834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,1536,0.08453120390574137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,768,0.06065599918365479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,256,0.02850773334503174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,16384,64,0.02030293345451355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,65536,1.717303466796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,16384,0.6559445063273113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,51200,1.3562240600585938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,16384,0.4584351857503255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,16384,0.5008330663045247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,65536,2.0979466756184895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,51200,2.3841280619303387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,12288,0.3455456097920736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,65536,3.0345130920410157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,12288,0.5478378931681316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,51200,1.6949834187825519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,12288,0.3923370679219564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,10240,0.47008641560872394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,10240,0.32410345077514646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,10240,0.29304320017496743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,8192,0.2434879938761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,8192,0.37217601140340173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,7168,0.31921812693277996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,8192,0.2795818646748861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,7168,0.21402986844380698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,7168,0.2389237403869629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,6144,0.18426027297973632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,6144,0.26027199427286785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,5120,0.15753919283548992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,5120,0.23137920697530112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,6144,0.20178346633911132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,5120,0.17487252553304036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,4096,0.125981863339742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,4096,0.14416213035583497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,4096,0.17484693527221679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,3584,0.1207477331161499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,3584,0.12486720085144043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,3072,0.1363701343536377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,3072,0.1007424036661784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,2560,0.11391572952270508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,3072,0.11076906522115071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,2560,0.08883732954661051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,2560,0.09371840159098307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,2048,0.09603520234425864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,1536,0.07331733703613282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,2048,0.08165547053019205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,1024,0.054100267092386876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,1536,0.06344000101089478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,1024,0.05431893269220987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,768,0.04357120196024577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,1024,0.05087893406550089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,768,0.04383999903996785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,512,0.0337066650390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,768,0.04537706772486369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,256,0.02369813323020935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,512,0.03856319983800252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,256,0.03225920001665751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,128,0.017800533771514894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,256,0.03399253288904826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,64,0.01469013293584188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,128,0.029983999331792195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,32,0.016429866353670754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,64,0.030268800258636475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,32,0.029994666576385498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,12288,3584,0.15368000666300458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,2048,0.07428159713745117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,512,0.03671893278757731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,16384,65536,3.1984512329101564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,12288,128,0.028304000695546467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,65536,1.5216927846272787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,65536,2.487852732340495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,51200,1.1617664337158202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,51200,1.3460853576660157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,65536,1.8620981852213543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,16384,0.5723285039265951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,16384,0.39236052831014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,16384,0.4264320055643718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,51200,1.9952064514160157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,12288,0.5225536028544109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,12288,0.34831040700276694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,12288,0.33541653951009115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,10240,0.25563093821207683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,8192,0.2825344085693359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,10240,0.35854934056599935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,10240,0.29220587412516275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,8192,0.2139509359995524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,8192,0.22564053535461426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,7168,0.26715946197509766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,7168,0.18410239219665528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,7168,0.1972917397816976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,6144,0.23000853856404624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,6144,0.16944853464762372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,6144,0.1789375940958659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,5120,0.18429759343465169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,5120,0.1459573268890381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,5120,0.13980053265889486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,4096,0.1172714630762736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,4096,0.1254688024520874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,3584,0.1009226640065511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,3072,0.11939520041147869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,3072,0.09524160226186117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,3072,0.09721279939015706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,2560,0.09553066889444986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,2560,0.0801141341527303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,2560,0.08480532964070639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,2048,0.08174506823221842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,2048,0.06730559666951498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,1536,0.06642560164133707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,1536,0.05790826479593912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,1536,0.056149331728617344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,1024,0.04691733519236247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,1024,0.04482773145039876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,768,0.03829760154088338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,768,0.04083626667658488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,768,0.04038506746292114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,512,0.029164799054463703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,512,0.034065067768096924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,512,0.03342293500900269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,256,0.019393066565195717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,256,0.02864426573117574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,256,0.028493867317835493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,128,0.014475733041763306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,128,0.027013333638509114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,128,0.02625280022621155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,64,0.013366400202115377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,32,0.013779200116793313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,64,0.026494934161504106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,32,0.026498132944107057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,4096,0.14879679679870605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,10240,3584,0.1327946662902832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,3584,0.10627199808756511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,10240,2048,0.07039039929707845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,10240,1024,0.046410667896270755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,16384,0.47273705800374344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,65536,1.273737589518229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,16384,0.3436095873514811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,51200,1.5448895772298177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,65536,2.0885951995849608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,12288,0.3625717480977376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,12288,0.2668383916219076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,10240,0.30258452097574867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,10240,0.21940266291300453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,8192,0.24816746711730958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,10240,0.31355625788370767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,51200,0.9835968017578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,8192,0.1730720043182373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,7168,0.20711679458618165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,7168,0.15014079411824544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,12288,1536,0.061793065071105956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,7168,0.22516907056172691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,6144,0.18117653528849284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,6144,0.12951467037200928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,5120,0.15689919789632162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,5120,0.11249813238779705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,5120,0.16058026949564616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,4096,0.12401599884033203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,4096,0.09854400157928467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,3584,0.10814507007598877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,3584,0.08661440213521322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,16384,0.4789888064066569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,3072,0.09383893013000488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,3072,0.07841599782307943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,51200,1.6882347106933593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,12288,0.3544447898864746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,2560,0.07950293223063151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,65536,2.397247060139974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,2560,0.06735573609670004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,2048,0.06597866614659628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,2048,0.05762346585591634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,2048,0.06394346555074057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,8192,0.2564256032307943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,1536,0.0509984016418457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,1536,0.04889813264211019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,1024,0.03632746537526448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,1024,0.039435732364654544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,6144,0.18840959866841633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,768,0.02956053415934245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,768,0.03487679958343506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,768,0.030756266911824544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,512,0.022859734296798707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,512,0.030801065762837726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,3072,0.0927232027053833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,4096,0.13016533056894938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,256,0.01656000018119812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,256,0.020835200945536293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,128,0.012645333011945089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,2560,0.08363839785257975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,128,0.01722453236579895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,64,0.011068800091743469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,64,0.02406186660130819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,8192,32,0.011522133151690166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,32,0.024408533175786336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,256,0.027918932835261027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,8192,128,0.02384106715520223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,65536,1.6841952006022136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,1536,0.047695998350779215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,65536,1.0891979217529297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,51200,1.1824223836263021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,1024,0.037829331556955975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,3584,0.11297706762949626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,16384,0.39261013666788735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,16384,0.3031968116760254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,12288,0.28213014602661135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,12288,0.22158506711324058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,51200,0.8576682408650717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,10240,0.23680639266967773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,12288,0.3136351903279623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,10240,0.181551996866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,8192,0.2019733270009359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,10240,0.2703392028808594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,8192,0.15238399505615235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,8192,0.2219775994618734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,7168,0.17843413352966309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,51200,1.3601759592692058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,7168,0.12919466495513915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,6144,0.14762239456176757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,6144,0.12076053619384766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,5120,0.12324906984965008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,6144,0.15799253781636555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,5120,0.09848106702168782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,4096,0.09961919784545899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,5120,0.14311253229777018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,4096,0.08554453055063883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,3584,0.08816959857940673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,4096,0.11507626374562581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,3584,0.08035199642181397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,3072,0.076473601659139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,3072,0.06803413232167563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,3072,0.08268053531646728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,2560,0.06477226813634238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,8192,512,0.024851200977961223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,2560,0.05904213190078735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,2048,0.053181866804758705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,1536,0.04166080156962077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,2048,0.05640319983164469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,1536,0.0434058666229248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,1024,0.030502400795618695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,1024,0.03531413475672404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,1024,0.033181866010030106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,768,0.02499413291613261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,768,0.03256319959958394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,2048,0.05126506487528483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,512,0.019453867276509603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,512,0.027930667002995808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,512,0.02180160085360209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,16384,0.4135029474894206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,256,0.01406613290309906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,256,0.01807253360748291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,65536,1.8847605387369792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,128,0.010947199662526448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,128,0.015313067038853965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,64,0.010134399930636088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,128,0.024245333671569825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,7168,32,0.011059199770291645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,64,0.023114667336146037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,32,0.024243199825286867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,7168,0.1927018642425537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,3584,0.09653440316518148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,2560,0.07183787027994791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,7168,256,0.024858667453130086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,65536,0.9790496190388998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,1536,0.04200746615727742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,65536,1.3835316975911458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,16384,0.3267178535461426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,65536,1.076913070678711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,7168,768,0.026718932390213012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,16384,0.29644266764322913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,51200,0.7645493189493815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,51200,1.0533717473347983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,12288,0.27707732518514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,16384,0.2628671964009603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,51200,0.8370229085286459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,8192,0.16718400319417318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,10240,0.21053333282470704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,10240,0.18124267260233562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,10240,0.179585059483846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,8192,0.15647786458333332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,7168,0.1488757292429606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,8192,0.1437290668487549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,7168,0.12768747011820475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,7168,0.13277333577473957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,6144,0.14344852765401203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,6144,0.10725759665171306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,6144,0.1132309357325236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,5120,0.10598613421122234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,5120,0.09018346468607584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,4096,0.08878400325775146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,5120,0.09721919695536295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,4096,0.07840960025787354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,4096,0.07903573513031006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,3584,0.07547413508097331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,3584,0.0724671999613444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,3584,0.07188373406728109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,3072,0.06207893292109171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,3072,0.06186453501383463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,2560,0.05765866835912069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,2560,0.05880106687545776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,2560,0.052919467290242515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,2048,0.04542933305104573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,1536,0.035835735003153485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,2048,0.04839893182118733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,1536,0.03803093433380127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,1024,0.026200532913208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,1536,0.03601386547088623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,768,0.0207370658715566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,1024,0.03100053270657857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,768,0.02834133307139079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,768,0.025268266598383587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,512,0.016148266196250916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,512,0.024895999828974405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,12288,0.1903157393137614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,512,0.02177813251813253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,256,0.01120746632417043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,128,0.008552533388137818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,256,0.022190932432810465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,12288,0.20655892690022787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,128,0.020413867632548013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,64,0.007457066575686138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,32,0.0077919999758402515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,64,0.019960532585779824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,6144,32,0.020518400271733604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,6144,3072,0.06598399877548218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,1024,0.029449599981307983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,65536,0.864073626200358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,65536,1.092066192626953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,128,0.016140799721082053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,16384,0.31133546829223635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,51200,0.6744810740152996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,16384,0.22739307085673013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,51200,0.9071264266967773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,12288,0.17268266677856445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,12288,0.20531093279520668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,65536,1.2945290883382161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,10240,0.14379199345906574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,10240,0.17375253041585287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,8192,0.13861653010050456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,8192,0.11490133603413899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,7168,0.12310400009155273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,7168,0.10248640378316243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,7168,0.14695040384928387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,6144,0.1062058687210083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,6144,0.09102079868316651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,16384,0.3102975845336914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,51200,1.0415231704711914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,12288,0.23586559295654297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,256,0.0175327996412913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,10240,0.20146986643473305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,4096,0.07597973346710205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,5120,0.09027307033538819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,4096,0.07095466454823812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,3584,0.07365547021230062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,8192,0.1700597286224365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,3584,0.06351999839146932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,3584,0.07335147062937418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,3072,0.05733760197957357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,2560,0.04905600150426229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,3072,0.056043732166290286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,2560,0.04899199803670247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,2560,0.05341226657231649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,2048,0.040814932187398276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,2048,0.04153813521067302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,1536,0.03144533236821492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,6144,0.12555413246154784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,1536,0.03557973305384318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,5120,0.08885973294576009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,1024,0.022539732853571574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,1024,0.0293013334274292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,1024,0.026586665709813433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,768,0.018199467658996583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,768,0.02669546604156494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,768,0.020116267601648967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,512,0.01432213286558787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,512,0.02387839953104655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,6144,2048,0.04275306860605876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,256,0.009960533181826273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,256,0.021237333615620933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,5120,0.10701013406117757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,128,0.007828266421953837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,4096,0.08435733318328857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,128,0.02000853419303894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,64,0.0070602665344874065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,64,0.0199178675810496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,5120,32,0.007572266459465027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,3072,0.061508266131083164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,5120,32,0.019925334056218467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,2048,0.043221334616343185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,1536,0.031922133763631184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,65536,0.86911252339681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,65536,0.7629781087239583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,16384,0.2226794719696045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,51200,0.6908661524454753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,65536,1.0496255874633789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,16384,0.20563839276631674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,51200,0.5910762786865235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,512,0.016638933618863424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,12288,0.15590826670328778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,12288,0.19325653711954754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,10240,0.13897066116333007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,51200,0.8437354405721029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,10240,0.13029440244038898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,8192,0.11166826883951823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,10240,0.16825599670410157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,8192,0.10828159650166828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,256,0.013392000397046407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,7168,0.09053546587626139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,6144,0.08654080231984457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,6144,0.08060479958852132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,6144,0.10216746330261231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,5120,0.07137386798858643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,5120,0.07149866422017416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,5120,0.08679680029551187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,4096,0.05874880154927572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,4096,0.05795199871063232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,4096,0.07235519886016846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,3584,0.05071893135706583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,3584,0.05292479991912842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,3584,0.06173866589864095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,3072,0.04444266557693481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,12288,0.17586453755696613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,3072,0.04601600170135498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,2560,0.0365941325823466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,5120,128,0.011693867047627766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,2560,0.04027413527170817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,2560,0.04633599917093913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,2048,0.030159999926884968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,1536,0.02330346703529358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,2048,0.03615359862645467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,1536,0.029206399122873945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,1024,0.016978132724761962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,1024,0.027053866783777875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,7168,0.09847040176391601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,768,0.013805866241455078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,768,0.02405760089556376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,768,0.01821546753247579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,512,0.01225600043932597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,512,0.02193280061086019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,2048,0.03487146695454915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,256,0.008572799960772197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,256,0.019719467560450236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,16384,0.2548746744791667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,256,0.012501333157221475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,128,0.006830933193365733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,128,0.018525866667429607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,128,0.011106133460998535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,64,0.006285866598288219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,4096,32,0.0064735998709996535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,64,0.01885973413785299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,4096,32,0.01880106727282206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,8192,0.13807786305745443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,7168,0.12227413654327393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,65536,0.7427413304646809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,65536,0.6879829406738281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,51200,0.5907007853190105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,3072,0.051614932219187414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,16384,0.20507946014404296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,16384,0.18620160420735676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,1536,0.02704213261604309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,51200,0.552785046895345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,12288,0.1443562666575114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,12288,0.13710932731628417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,1024,0.021944532791773476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,10240,0.12138880093892415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,10240,0.11401173273722331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,8192,0.0945685307184855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,8192,0.09251733620961508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,4096,512,0.014845866958300272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,7168,0.08416106700897216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,7168,0.08364906311035156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,8192,0.12369600137074788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,6144,0.07178346316019693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,6144,0.07488853136698405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,65536,0.9679360071818033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,5120,0.06367466847101846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,51200,0.7687157313028972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,12288,0.18384960492451985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,5120,0.06939733028411865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,4096,0.05010879834493002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,4096,0.05843413273493449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,3584,0.04452159802118937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,10240,0.15968106587727865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,3584,0.05208746592203776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,3072,0.03866879940032959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,3072,0.04482666651407878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,2560,0.03323093255360921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,2560,0.03969706694285075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,7168,0.11222399870554607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,2048,0.026872533559799194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,2048,0.03365120093027751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,16384,0.23857599894205728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,1536,0.02177600065867106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,1536,0.029418667157491047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,6144,0.09648853143056234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,1024,0.016387200355529784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,1024,0.025702399015426636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,1024,0.02056000034014384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,768,0.013878400127092997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,768,0.02293013334274292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,5120,0.08101653258005778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,512,0.011033599575360615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,4096,0.06652799844741822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,512,0.021497599283854165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,3584,0.05790826479593912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,512,0.014386133352915446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,256,0.008080000181992848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,3072,0.0485429326693217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,128,0.006706133484840393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,256,0.019092265764872232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,128,0.018281600872675576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,64,0.01840959986050924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,32,0.006185600161552429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3584,32,0.018346667289733887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,2560,0.043305599689483644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,2048,0.033982932567596436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,65536,0.6439744313557942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,65536,0.6364800135294597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,1536,0.025111466646194458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3584,64,0.005897599955399831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,65536,0.6160501480102539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,51200,0.5524650573730469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,51200,0.5006229400634765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,16384,0.16175467173258465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,16384,0.18167786598205565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,51200,0.47523094813028977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,16384,0.16567893028259278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,12288,0.1224554697672526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,12288,0.12707733313242595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,10240,0.10127039750417073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,12288,0.1257525364557902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,8192,0.08289173444112143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,10240,0.11413439909617107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,10240,0.10471253395080567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,8192,0.09061973094940186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,8192,0.08757973512013753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,7168,0.07311360041300455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,7168,0.08070933024088542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,6144,0.06269333362579346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,6144,0.07234133084615071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,5120,0.05300480127334595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,768,0.016825600465138753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,5120,0.06369173526763916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,4096,0.043299198150634766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,5120,0.05620479981104533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,4096,0.05037013292312622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,4096,0.04780906836191813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,3584,0.03821440140406291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,3584,0.045067731539408365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,3584,0.04076480070749919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,3072,0.04149866501490275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,2560,0.02791999975840251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,2560,0.03662613232930501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,2048,0.0236842672030131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,2048,0.032313599189122515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,2048,0.027194666862487792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,1536,0.019076265891393027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,1536,0.028990934292475384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,1536,0.022749867041905722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,256,0.011590400338172912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,1024,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,1024,0.017565866311391197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3584,128,0.010451199611028035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,768,0.012263466914494831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,768,0.016006400187810264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,6144,0.06478079954783121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,512,0.009484799702962239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,512,0.020755199591318767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,3072,0.03439466555913289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,256,0.007142400244871776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,3072,0.03585600058237712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,256,0.011653332908948263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,256,0.018481065829594932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,128,0.005901866654555003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,128,0.017463467518488564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,128,0.010803199807802836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,64,0.005305600166320801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,3072,32,0.005692799886067709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,64,0.017544533809026083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,32,0.018179200092951455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,1024,0.0236842672030131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,3072,768,0.022249599297841392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,65536,0.5449759801228841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,65536,0.5863125483194987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,51200,0.42192853291829424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,512,0.013857066631317139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,65536,0.6732906977335612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,51200,0.47565546035766604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,16384,0.1386474609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,51200,0.513267199198405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,12288,0.1032480001449585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,12288,0.12672959963480632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,16384,0.15711359977722167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,16384,0.17212479909261066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,12288,0.12979413668314616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,10240,0.08692906697591146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,8192,0.0738602638244629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,8192,0.0871562639872233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,7168,0.06349013249079385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,7168,0.07593386967976888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,6144,0.05478506485621134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,6144,0.06677866776784261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,6144,0.06535146633783975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,5120,0.04564693371454875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,5120,0.0573365330696106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,4096,0.037336532274882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,4096,0.04636160135269165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,4096,0.04608000119527181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,10240,0.10675733089447022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,3584,0.04201813141504924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,7168,0.07863252957661947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,3072,0.02942720055580139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,3072,0.037979733943939206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,3584,0.03293973406155904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,2560,0.024685867627461753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,2560,0.033613868554433185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,10240,0.11050026416778565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,8192,0.08875093460083008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,2048,0.020631466309229532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,2048,0.029899734258651733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,1536,0.01693120002746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,1536,0.026733867327372235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,2560,0.03096853295962016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,5120,0.055795200665791835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,1024,0.012914133071899415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,1024,0.01609280010064443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,1024,0.02195840080579122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,768,0.010441600282986959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,768,0.021441066265106203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,768,0.013029332955678305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,512,0.00842026670773824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,512,0.01978666583697001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,512,0.011174399654070537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,256,0.00647680014371872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,256,0.018326399723688762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,256,0.008989866574605305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,128,0.005555200080076853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,128,0.01766506632169088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,64,0.004950400193532308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,3072,7168,0.07544213136037191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2560,32,0.005301333467165629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,64,0.017552000284194947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,3584,0.040166401863098146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2560,32,0.017578667402267455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,3072,0.033843199412028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,65536,0.4291797320048015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,65536,0.5353557586669921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,51200,0.3481418609619141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,16384,0.11375466982523601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,51200,0.42238400777180984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,16384,0.14408000310262042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,12288,0.08630719979604086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,12288,0.11043519973754883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,2560,0.02983679970105489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,10240,0.07107093334197997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,2048,0.024333866437276204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,10240,0.09123626550038656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,8192,0.05756800174713135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,1536,0.019364267587661743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,8192,0.07511573632558187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,7168,0.050802131493886314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,7168,0.06754666964213053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2560,128,0.008091733356316884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,6144,0.04327679872512817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,6144,0.05937066475550333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,12288,0.12513173421223958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,5120,0.03760319948196411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,5120,0.05048426787058512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,65536,0.6659978866577149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,4096,0.030689066648483275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,16384,0.172490660349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,4096,0.04257706801096599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,3584,0.02746666669845581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,3584,0.03801706631978353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,10240,0.11091840267181396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,3072,0.023169066508611044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,3072,0.03383573293685913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,51200,0.5210282643636067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,2560,0.02068586746851603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,2560,0.03174186746279399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,7168,0.07634133497873942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,2048,0.017087999979654947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,2048,0.02661120096842448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,8192,0.0845461368560791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,1536,0.013843199610710144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,1536,0.023886932929356893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,6144,0.06359573205312094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,1024,0.010428800185521444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,1024,0.021126399437586464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,3072,0.03310719927151998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,768,0.008821333448092144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,768,0.020118399461110433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,4096,0.045162665843963626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,512,0.007330133517583211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,512,0.019153066476186118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,1536,0.018949333826700845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,256,0.005880533158779145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,5120,0.05445226828257242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,256,0.017826133966445924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,128,0.004997333387533823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,128,0.017262933651606242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,2560,0.030186667044957476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,64,0.004492799937725067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,64,0.01737706661224365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,2048,32,0.004715733230113983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,3584,0.038734932740529374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,2048,32,0.01729066570599874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,65536,0.33423999150594075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,65536,0.5040885289510091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,65536,0.38458026250203453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,2048,0.023856000105539957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,51200,0.26075946489969887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,16384,0.08935786883036295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,51200,0.30065279006958007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,16384,0.13224533398946126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,51200,0.3909685452779134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,12288,0.06706559658050537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,16384,0.10378452936808269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,12288,0.10108479658762615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,10240,0.06665173371632895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,12288,0.0803551991780599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,10240,0.08673280080159505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,10240,0.0681717316309611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,8192,0.07015039920806884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,7168,0.0470090667406718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,7168,0.06223040024439493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,7168,0.04776639938354492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,6144,0.040504535039265946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,6144,0.0546069343884786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,8192,0.04899839957555135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,5120,0.03462506532669067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,5120,0.04585493405659993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,5120,0.03640640179316203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,4096,0.026834134260813398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,4096,0.03866453170776367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,1024,0.01511146624883016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,4096,0.030114134152730305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,3584,0.023517866929372154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,3584,0.03482346534729004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,3584,0.02661973237991333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,3072,0.02066133419672648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,3072,0.03204693396886189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,2560,0.017398399114608765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,2560,0.028653866052627562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,128,0.007925333579381307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,2048,0.016378666957219443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,256,0.008728532989819845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,2048,0.025857067108154295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,2048,0.01816213329633077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,1536,0.012534399827321371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,1536,0.0155157337586085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,1536,0.02241599957148234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,1024,0.00990613301595052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,1024,0.02031893332799276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,768,0.007852800190448761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,768,0.019691733519236247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,512,0.011037866274515789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,768,0.011480533083279927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,512,0.0064298664530118305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,512,0.018179200092951455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,256,0.005081599950790406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,256,0.017037866512934367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,256,0.008507733543713886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,2048,768,0.0127893328666687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,128,0.004412800073623657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,128,0.0160181333621343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,128,0.00783253312110901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,64,0.003964799890915553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,64,0.016226133704185484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1536,32,0.004189866781234741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1536,32,0.01628266672293345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,65536,0.22776533762613932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,65536,0.43632319768269856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,65536,0.43969173431396485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,51200,0.1811573346455892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,51200,0.35603628158569334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,16384,0.06281280120213827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,16384,0.1196874698003133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,2560,0.02103040019671122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,12288,0.05059306621551514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,51200,0.3394890785217285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,12288,0.09116266568501791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,8192,0.053617068131764735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,10240,0.040055465698242185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,6144,0.042046932379404704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,10240,0.07627413272857667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,8192,0.03243733247121175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,8192,0.060354133447011314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,8192,0.06074986855189005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,512,0.010145066181818645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,7168,0.028985599676767986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,7168,0.05346879959106445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,6144,0.024690133333206177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,6144,0.04581760168075562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,5120,0.020721065998077392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,5120,0.03880319992701213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,16384,0.11580373446146648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,4096,0.017373865842819212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,4096,0.03360106547673543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,4096,0.031031467517217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,3584,0.015702399611473083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,3584,0.030322132507960002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,3072,0.023893332481384276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,3072,0.013913599650065103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,3072,0.028123732407887774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,3072,0.02368320027987162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,2560,0.012126933534940083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,2560,0.02656853397687276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1536,1024,0.01266986628373464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,12288,0.0843178669611613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,2048,0.010108799735705058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,2048,0.023774933815002442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,10240,0.0712650696436564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,1536,0.008497066299120585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,1536,0.021127466360727945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,1024,0.006842666864395141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,7168,0.05173866748809815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,1024,0.019057067235310872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,768,0.006001066664854685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,1024,0.010899200042088827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,6144,0.04473066727320353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,768,0.018758400281270345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,512,0.005160533388455709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,512,0.01781439979871114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,256,0.004333866635958353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,5120,0.03668053150177002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,256,0.016812799374262492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,128,0.00425493319829305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,128,0.0156960000594457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,3584,0.02800106604894002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,64,0.0038560000558694207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,64,0.01646080017089844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,1024,32,0.003913599997758865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,1024,32,0.015768532951672874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,65536,0.1774677276611328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,65536,0.40610561370849607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,2560,0.02058666745821635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,51200,0.13752533594767252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,51200,0.3207445462544759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,2048,0.017108267545700072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,16384,0.050587733586629234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,1536,0.014109866817792258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,16384,0.12531946500142416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,768,0.009231999516487122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,12288,0.0591754674911499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,12288,0.09710720380147299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,512,0.008169599870840708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,10240,0.05514453252156576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,10240,0.08156586488087972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,8192,0.04090240001678467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,8192,0.06680853366851806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,128,0.006300800045331319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,7168,0.040830934047698976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,7168,0.057272533575693764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,1024,256,0.006908800204594929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,6144,0.030987733602523805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,6144,0.04732053279876709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,65536,0.32008320490519204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,5120,0.02706986665725708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,5120,0.04208639860153198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,51200,0.2514154593149821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,4096,0.021026132504145305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,4096,0.03609600067138672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,12288,0.06497600078582763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,3584,0.019001599152882895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,10240,0.05514453252156576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,3584,0.03140799999237061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,8192,0.0454367995262146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,3072,0.01702186663945516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,3072,0.0288917342821757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,7168,0.04030826489130656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,2560,0.015842133760452272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,2560,0.027221333980560303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,16384,0.08592853546142579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,2048,0.011975466211636861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,2048,0.023873066902160643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,6144,0.03471039930979411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,1536,0.009585066636403402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,1536,0.022205867369969687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,1024,0.007347199817498524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,1024,0.019780266284942626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,5120,0.029842134316762286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,768,0.00639466643333435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,768,0.018261333306630455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,4096,0.025246934096018477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,512,0.005111466844876607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,512,0.017573332786560057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,3072,0.019937066237131755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,256,0.004178133110205332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,256,0.01632960041364034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,2048,0.015145599842071533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,128,0.003940266619126002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,3584,0.02290346622467041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,128,0.01585706671079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,64,0.003517866631348928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,64,0.01556373337904612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,768,32,0.0035786665976047516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,768,32,0.015607466300328573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,65536,0.12576853434244792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,2560,0.017734400431315103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,65536,0.39632425308227537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,51200,0.12914666334788005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,51200,0.31007254918416344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,1536,0.01330560048421224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,16384,0.041773867607116696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,16384,0.10909226735432942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,1024,0.010473600029945374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,12288,0.034533333778381345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,12288,0.08353066444396973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,768,0.009193600217501322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,10240,0.045655465126037596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,512,0.00867306689421336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,10240,0.06922453244527181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,8192,0.02224853237469991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,65536,0.3447967847188314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,8192,0.054039466381073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,7168,0.020092799266179403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,256,0.007294933497905731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,7168,0.0468234658241272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,6144,0.01755733291308085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,6144,0.04050986766815186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,6144,0.03605653444925944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,5120,0.015321600437164306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,768,128,0.006680533289909363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,5120,0.035641598701477054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,5120,0.03092479904492696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,4096,0.013052800297737121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,4096,0.0304149329662323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,3584,0.01183893382549286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,3584,0.02802666624387105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,51200,0.27178239822387695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,3072,0.010314666231473287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,3072,0.026334933439890545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,16384,0.09078186353047689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,2560,0.009201066692670186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,2560,0.02386666735013326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,12288,0.0710208018620809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,2048,0.008042666812737782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,2048,0.02190399964650472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,10240,0.05873706738154093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,1536,0.006997333467006683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,8192,0.04856853485107422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,1536,0.02039360006650289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,1024,0.0058783998092015585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,1024,0.019827200969060262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,7168,0.04217706521352132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,768,0.005224533379077911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,768,0.0180021325747172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,768,0.00795306662718455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,512,0.004548266530036926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,512,0.016781866550445557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,4096,0.02527466615041097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,256,0.0038272000849246977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,256,0.01616320013999939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,3584,0.022729599475860597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,128,0.0034282666941483817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,128,0.015126400192578635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,3072,0.01986666719118754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,64,0.0032746667663256326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,64,0.01562773287296295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,512,32,0.0032831999162832894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,512,32,0.015543466806411744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,65536,0.0829802672068278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,2560,0.017343999942143758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,65536,0.3602239926656087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,2048,0.015010133385658264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,51200,0.06758506298065185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,51200,0.2916085243225098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,1536,0.012151466806729634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,16384,0.031190399328867597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,16384,0.10257919629414876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,1024,0.009405866265296936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,12288,0.02525866627693176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,12288,0.07871786753336588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,512,0.007119999825954437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,256,0.006041599810123444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,10240,0.01862186590830485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,10240,0.06408640146255493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,8192,0.016239999731381734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,512,128,0.005452799797058106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,8192,0.04882560173670451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,7168,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,7168,0.042453332742055254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,65536,0.31488107045491537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,6144,0.013123200337092081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,6144,0.036697598298390706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,5120,0.011606400211652119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,5120,0.03285226623217265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,51200,0.25721920331319176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,4096,0.010126933455467224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,16384,0.0849461317062378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,4096,0.02813653349876404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,3584,0.009457066655158997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,3584,0.027246934175491334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,12288,0.06468799908955893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,3072,0.008134399851163227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,3072,0.024641066789627075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,10240,0.05465813477834066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,2560,0.007315200070540111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,8192,0.0439850648244222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,2560,0.022805333137512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,2048,0.00660159985224406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,2560,0.01590079963207245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,2048,0.021150932709376017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,7168,0.0387392004330953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,1536,0.005673600236574808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,1536,0.01128426690896352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,1536,0.020077866315841675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,1024,0.004816000163555145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,1024,0.01783039967219035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,768,0.0044383997718493145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,768,0.01709653337796529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,6144,0.03338453372319539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,512,0.003819733361403147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,512,0.017017600933710735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,5120,0.028515199820200603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,256,0.0033919999996821085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,256,0.01595200002193451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,4096,0.023494400580724082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,128,0.0031167998909950255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,128,0.015549866358439126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,3584,0.021166932582855225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,64,0.002932266642649968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,3072,0.018389334281285606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,64,0.016310399770736693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,256,32,0.0030303999781608583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,256,32,0.015096533298492431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,65536,0.06998186906178792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,2048,0.013661866386731466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,65536,0.23371200561523436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,65536,0.3504298528035482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,51200,0.05662399927775065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,16384,0.023066665728886923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,51200,0.18294399579366047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,51200,0.2784245491027832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,16384,0.09779626528422038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,16384,0.06253120104471842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,768,0.0077919999758402515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,12288,0.015759999553362526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,12288,0.07594026724497477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,10240,0.013422933220863343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,10240,0.06117333173751831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,8192,0.01162986656030019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,10240,0.04102293252944946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,7168,0.010591999689737955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,8192,0.04682026704152425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,7168,0.04029866854349772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,6144,0.011688533425331115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,6144,0.035224533081054686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,6144,0.02540480097134908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,5120,0.010102400183677673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,5120,0.031352533896764116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,512,0.006843733290831248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,4096,0.00869866708914439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,4096,0.028091732660929365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,256,0.005737600227197012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,3584,0.007922133306662242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,1024,0.009020800391832989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,3584,0.02540266712506612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,3072,0.00735040009021759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,3072,0.024228266874949136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,256,128,0.00525546669960022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,2560,0.006466133395830791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,2560,0.022363734245300294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,2560,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,12288,0.04791680177052816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,2048,0.005707733333110809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,2048,0.021643733978271483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,8192,0.033405868212382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,1536,0.005238399902979533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,7168,0.02910933295885722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,1536,0.018931200106938682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,1024,0.004533333579699198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,1024,0.018293333053588868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,1024,0.00853546659151713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,768,0.004075733323891958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,768,0.017388800779978432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,5120,0.021720532576243082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,512,0.0036042665441830954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,512,0.016198399662971496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,4096,0.018819200992584228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,256,0.003304533412059148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,256,0.015503999590873719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,3584,0.01712426741917928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,128,0.0029557332396507262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,3072,0.015227733055750528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,128,0.005580799778302511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,128,0.01535040040810903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,64,0.002869333326816559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,128,32,0.0029909332593282064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,64,0.015029333035151162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,128,32,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,65536,0.06714666684468587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,51200,0.053174400329589845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,65536,0.3474634806315104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,51200,0.27499732971191404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,16384,0.02044266661008199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,12288,0.011891200145085653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,16384,0.09659626483917236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,10240,0.010795733332633973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,8192,0.009608532985051472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,10240,0.05806933244069418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,8192,0.04390613238016765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,7168,0.00902506709098816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,6144,0.00897920032342275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,7168,0.038254932562510176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,5120,0.007866666714350382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,6144,0.03425813515981038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,4096,0.007316266496976216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,5120,0.030103466908137005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,3584,0.0069013332327206925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,4096,0.027035733064015705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,3584,0.025192532936731977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,3072,0.006846933563550313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,2560,0.0062047998110453285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,3072,0.02390186587969462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,2560,0.022193066279093423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,2048,0.005414400001366933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,1536,0.004834133386611939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,2048,0.02055466572443644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,1024,0.004166399935881296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,1024,0.0183242658774058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,768,0.0038634667793909705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,2048,0.011547733346621196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,768,0.016989866892496742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,1536,0.010477866729100544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,512,0.003549866626660029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,256,0.0031210665901501974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,512,0.016243199507395424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,256,0.015383467078208923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,128,0.002882133424282074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,64,0.002881066749493281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,128,0.014677332838376364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,64,32,0.0028480000793933867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,64,0.014728533228238425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,12288,0.07174719969431559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,32,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,65536,0.07097919782002768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,16384,0.018796799580256145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,51200,0.054840532938639316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,12288,0.012407466769218445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,16384,0.09665599664052328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,768,0.007573333382606506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,10240,0.010628267129262289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,12288,0.07110826969146729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,51200,0.27481813430786134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,8192,0.009302399555842082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,65536,0.34901866912841795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,7168,0.008805333574612936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,6144,0.007567999760309856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,512,0.007077333331108093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,8192,0.04335466623306274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,5120,0.007934933404127757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,4096,0.00693333347638448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,5120,0.03038826584815979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,3584,0.006681600213050842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1281,128,256,0.006128000219662985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,4096,0.02720639904340108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,3072,0.0071381335457166035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,3584,0.025146667162577314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,2560,0.006106666723887126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,3072,0.023590399821599325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,2048,0.005407999952634176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,1536,0.004799999793370565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,2560,0.02230293353398641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,1024,0.004110933343569437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,1024,0.017403733730316163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,768,0.0037834666669368743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,64,1536,0.019269333283106486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,768,0.01663253307342529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,512,0.00348693331082662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,256,0.0031317333380381264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,512,0.016110933820406594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,128,0.0028597332537174227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,128,0.01495039959748586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,64,0.002734933296839396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,64,0.014669866363207499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1281,32,32,0.0027285332481066385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,32,0.014791466792424521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,10240,0.057447465260823574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,7168,0.03737920125325521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,6144,0.033056000868479415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,2048,0.020866133769353232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,1536,0.019301333030064902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1281,32,256,0.01567359964052836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,16384,1.937282180786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,16384,2.364984639485677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,12288,2.948564147949219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,12288,1.4308619181315103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,16384,3.984185536702474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,12288,1.845412317911784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,10240,1.26407470703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,10240,2.353435770670573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,10240,1.5304704030354819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,8192,1.0157322565714517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,8192,1.8302954355875651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,8192,1.2364330291748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,7168,0.8828800201416016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,51200,6.47698974609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,7168,1.0740010579427084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,7168,1.6218346913655597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,6144,1.4495285034179688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,6144,0.806281598409017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,6144,0.9296672185262045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,5120,0.6866847991943359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,5120,0.7627797444661458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,4096,0.5633087793986002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,5120,1.212933349609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,4096,0.9320490519205729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,4096,0.6422965367635091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,3584,0.47284266153971355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,3584,0.8323306401570638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,3584,0.5525461196899414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,51200,8.704727172851562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,3072,0.4063071886698405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,3072,0.4831263860066731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,3072,0.7427871704101563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,2560,0.3621376037597656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,2560,0.6030037562052409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,2560,0.42424319585164383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,2048,0.2918485323588053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,2048,0.35456854502360025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,1536,0.23022933006286622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,1024,0.17090986569722494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,1024,0.25916694005330404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,2048,0.49418026606241866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,1536,0.39119253158569334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,1536,0.3085013389587402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,768,0.14257386525472004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,768,0.19628267288208007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,1024,0.22107839584350586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,512,0.12703359921773275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,512,0.15522665977478028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,512,0.16664106051127117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,768,0.20303786595662437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,256,0.09166613419850668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,256,0.13291306495666505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,64,0.07186133066813151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,128,0.0888533353805542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,128,0.08496639728546143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,65536,128,0.13060800234476727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,32,0.0708405335744222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,64,0.0851690689722697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,65536,32,0.08406506379445394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,256,0.09447893301645914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,65536,51200,11.865054321289062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,16384,2.0307498931884767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,16384,3.1205014546712238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,51200,5.269667053222657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,16384,1.9135594685872395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,12288,2.255441029866536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,51200,6.16781005859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,12288,1.1755157470703126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,65536,6.532006327311199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,10240,0.9979573567708334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,10240,1.9474068959554036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,10240,1.164894994099935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,65536,9.199172973632812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,8192,0.8421248118082681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,8192,1.5032117207845053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,51200,9.675602213541667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,8192,0.9457258860270181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,7168,1.375876235961914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,7168,0.683026123046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,6144,1.113274637858073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,6144,0.7148309071858724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,12288,1.3945120493570964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,5120,0.5030271848042805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,5120,0.9939893086751302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,5120,0.599016507466634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,4096,0.4275477409362793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,7168,0.7989791870117188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,4096,0.8373823801676432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,4096,0.4666144053141276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,3584,0.37658665974934896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,6144,0.6169216156005859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,3584,0.4107381184895833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,3584,0.7095349629720051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,3072,0.3230293273925781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,3072,0.6057109196980794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,3072,0.3748138745625814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,2560,0.27230186462402345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,2560,0.3327146530151367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,2048,0.2502773284912109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,2560,0.45554027557373045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,1536,0.29357547760009767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,2048,0.3105450630187988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,65536,12.51712646484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,1024,0.22051199277242026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,1536,0.18606826464335124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,1536,0.23923947016398112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,1024,0.18662293752034503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,1024,0.1432352066040039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,512,0.1241418679555257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,512,0.08887466589609781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,768,0.12946666876475016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,768,0.15057279268900553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,512,0.12990612983703614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,256,0.07105387051900228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,768,0.1757365385691325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,128,0.0601365327835083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,256,0.07440213362375894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,128,0.07063893477121988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,64,0.06567786534627279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,256,0.11986773014068604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,32,0.05719786485036214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,51200,128,0.10043839613596599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,64,0.06974720160166423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,51200,32,0.06964053312937418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,51200,2048,0.3913087844848633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,16384,0.8449994405110678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,16384,0.527942403157552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,16384,0.6007178624471028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,12288,0.6824437459309896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,51200,1.674872589111328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,51200,2.001210657755534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,12288,0.43802452087402344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,12288,0.45238507588704424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,10240,0.5734506607055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,65536,2.268349965413411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,10240,0.36747519175211585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,65536,2.5559552510579424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,10240,0.3985205332438151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,8192,0.48889067967732747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,51200,2.8540992736816406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,8192,0.2958869298299154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,8192,0.31626453399658205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,7168,0.2686026573181152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,7168,0.3973461469014486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,7168,0.28489920298258464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,6144,0.3496074676513672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,6144,0.23284692764282228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,6144,0.2236010710398356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,5120,0.18697387377421062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,5120,0.2906570752461751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,5120,0.19698346455891927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,4096,0.23480426470438637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,3584,0.13759573300679523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,4096,0.1632522741953532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,3584,0.15333760579427083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,3584,0.20235733985900878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,3072,0.17286186218261718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,3072,0.1263306697209676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,65536,3.808012898763021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,3072,0.12626559734344484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,2560,0.11393600304921467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,2048,0.12105387051900227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,2048,0.09011200269063314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,1536,0.0910271962483724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,1024,0.06684693495432535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,1536,0.07669119834899903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,1024,0.0588266650835673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,2048,0.10959893067677814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,768,0.05476266543070475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,1024,0.061610666910807285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,768,0.05149226586023966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,512,0.04102613528569539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,512,0.04401386578877767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,768,0.057366398970286045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,512,0.0458837350209554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,256,0.02910826603571574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,128,0.022804266214370726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,256,0.03619626760482788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,256,0.03843306700388591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,128,0.033905065059661864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,128,0.03567786614100139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,64,0.01990613341331482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,64,0.03391253153483073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,32,0.020402133464813232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,32,0.034541865189870194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,16384,4096,0.1646015961964925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,16384,2560,0.14616746902465821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,2560,0.11178773244222004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,16384,1536,0.08287466367085775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,16384,0.6376000086466471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,65536,1.745809046427409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,16384,0.4673343976338704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,51200,1.2837791442871094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,12288,0.33323628107706704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,12288,0.5007349332173665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,51200,2.1925280253092447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,65536,2.9692128499348955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,10240,0.44310614267985027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,10240,0.29647467931111654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,8192,0.33082987467447916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,8192,0.22804266611735025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,8192,0.26899305979410804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,7168,0.2915040016174316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,7168,0.196669864654541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,6144,0.17182933489481608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,6144,0.2645045280456543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,5120,0.22292052904764809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,5120,0.15899626413981122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,65536,2.2857706705729166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,51200,1.7507082621256511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,4096,0.20309759775797526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,16384,0.5150997479756673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,4096,0.14622186024983724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,3584,0.16582506497701008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,3584,0.11723626454671224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,12288,0.3833237330118815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,3072,0.1373301347096761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,3072,0.09646613597869873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,2560,0.10955946445465088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,10240,0.3560096104939779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,2560,0.08616853555043538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,2048,0.08980586528778076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,2048,0.07302613258361816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,7168,0.24499306678771973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,2048,0.07772266864776611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,1536,0.06941973368326823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,1536,0.061386664708455406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,1536,0.05896533330281576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,1024,0.05020586649576823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,1024,0.04946773449579875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,6144,0.1948863983154297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,768,0.04114133516947428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,768,0.043186132113138834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,512,0.03149226705233256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,512,0.037495466073354085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,5120,0.1713706652323405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,256,0.02243306636810303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,256,0.031684267520904544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,4096,0.14239253997802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,128,0.016956800222396852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,128,0.029227733612060547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,3584,0.12027200063069661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,64,0.014574933052062988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,64,0.029281065861384077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,3072,0.1005621353785197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,12288,32,0.01564479966958364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,12288,32,0.029088000456492107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,2560,0.09316693147023519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,1024,0.047990401585896805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,768,0.03834559917449951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,512,0.03147413333257039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,256,0.027005867163340254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,65536,1.464584477742513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,16384,0.38761491775512696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,16384,0.5708192189534504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,51200,1.135047403971354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,65536,2.5172767639160156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,12288,0.46476586659749347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,12288,0.2872725486755371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,51200,1.8615637461344403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,12288,128,0.022929066419601442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,10240,0.24340480168660483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,10240,0.39147733052571615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,8192,0.28426027297973633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,8192,0.19598933855692546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,7168,0.2505023956298828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,7168,0.1742421309153239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,6144,0.21556587219238282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,6144,0.1574207941691081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,12288,0.342199452718099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,5120,0.18003093401590983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,5120,0.13019946416219075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,65536,1.855237325032552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,16384,0.428494930267334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,4096,0.1801344076792399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,6144,0.16906026204427083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,4096,0.10964372952779133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,3584,0.12874666849772137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,3584,0.09887680212656656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,3072,0.11060372988382976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,10240,0.30480213165283204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,3072,0.08762239615122477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,7168,0.19947306315104169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,2560,0.09531946976979574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,2560,0.07737279733022054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,51200,1.3962880452473958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,2048,0.0920565366744995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,8192,0.22750933965047201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,2048,0.07494293053944906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,1536,0.07351253032684327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,1536,0.051520001888275144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,1024,0.038013867537180585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,1024,0.03887679974238078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,3072,0.08491840362548828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,768,0.03110719919204712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,768,0.034705066680908205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,4096,0.11718613306681316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,512,0.023015467325846355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,512,0.034689064820607504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,5120,0.14110933939615886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,256,0.02031360069910685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,256,0.031085866689682006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,2560,0.07749013106028238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,128,0.017220266660054526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,128,0.028761599461237592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,3584,0.09961706797281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,64,0.016280532876650492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,64,0.029527467489242554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,1536,0.04806613524754842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,10240,32,0.016823466618855795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,10240,32,0.02927146752675374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,2048,0.06312426726023355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,65536,1.1945706685384114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,65536,1.8047531127929688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,1024,0.0420085350672404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,51200,1.4587498982747396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,51200,0.9307477315266928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,51200,0.9412672042846679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,16384,0.3117173194885254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,16384,0.43562774658203124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,16384,0.3157898585001627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,12288,0.31969706217447913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,12288,0.2527402718861898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,12288,0.25445440610249836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,10240,0.2697087923685709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,10240,0.19994133313496906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,10240,0.210204807917277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,8192,0.15856320063273113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,8192,0.22152320543924967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,8192,0.1696128050486247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,7168,0.18761599858601888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,7168,0.14751680692036945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,7168,0.14916693369547526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,6144,0.164683739344279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,128,0.018311466773351034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,6144,0.12755093574523926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,256,0.0214303990205129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,5120,0.10316800276438395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,5120,0.10400853157043458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,4096,0.10985493659973145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,4096,0.08777600129445394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,3584,0.07972586949666341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,3584,0.07458133697509765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,512,0.025838933388392132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,3072,0.08533013661702474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,3072,0.0709706703821818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,3072,0.06528319915135702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,2560,0.07220799922943115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,10240,768,0.031666133801142374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,2560,0.06212799946467081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,2560,0.05753386815388998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,2048,0.059298133850097655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,2048,0.05379199981689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,2048,0.04915200074513753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,1536,0.04659946759541829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,1536,0.0453877329826355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,1024,0.03329600095748901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,1536,0.04110613266626994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,768,0.02640426754951477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,1024,0.033319467306137086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,1024,0.036082132657368975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,512,0.020347734292348228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,768,0.032638933261235556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,512,0.025050665934880572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,256,0.014382933576901754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,256,0.024487467606862386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,6144,0.12851200103759766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,128,0.011041067043940226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,256,0.02108586629231771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,4096,0.08559573491414388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,128,0.023031467199325563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,64,0.009193600217501322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,32,0.009511466821034749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,3584,0.09726186593373617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,32,0.022380799055099487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,768,0.029767467578252153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,512,0.028531199693679808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,8192,5120,0.14065814018249512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,128,0.01945919990539551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,65536,1.1298826853434245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,8192,64,0.023293866713841756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,65536,1.702902348836263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,16384,0.38918399810791016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,51200,0.8695850372314453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,16384,0.30917228062947594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,51200,1.2408373514811197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,12288,0.30740267435709634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,12288,0.23557653427124023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,10240,0.2600757280985514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,10240,0.19082454045613606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,8192,0.20033493041992187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,8192,0.1540544033050537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,8192,65536,1.2769332885742188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,7168,0.19903465906778972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,7168,0.15061012903849286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,6144,0.15784640312194825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,6144,0.12940693696339925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,16384,0.3033088048299154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,65536,1.272329584757487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,51200,1.0236426671346028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,5120,0.1151039997736613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,12288,0.23208212852478027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,4096,0.11569706598917644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,4096,0.0853877305984497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,3584,0.09256746768951415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,10240,0.1940949281056722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,3584,0.07766613165537516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,3072,0.08053226470947265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,3072,0.06931306521097819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,8192,0.15726826985677084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,2560,0.06724692980448405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,2560,0.06103893518447876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,5120,0.12621013323465985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,2048,0.05441919962565104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,2048,0.05314666827519735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,7168,0.13860373497009276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,1536,0.04322239955266317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,1536,0.044309333960215254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,6144,0.121833602587382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,1024,0.03219946622848511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,1024,0.036673065026601157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,5120,0.1015455961227417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,768,0.026182399193445845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,4096,0.08283092975616455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,768,0.033572268486022946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,512,0.020523732900619505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,512,0.029402667284011842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,3584,0.07241493066151937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,256,0.014436266819636025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,256,0.024709333976109825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,3072,0.06303786834081014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,128,0.011530666550000509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,128,0.023176532983779908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,2560,0.05605440139770508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,64,0.010542933146158855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,2048,0.045823999245961505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,64,0.02341759999593099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,7168,32,0.011699199676513672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,7168,32,0.02334400018056234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,1536,0.03658560117085775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,65536,0.9715434392293295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,65536,1.396021270751953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,1024,0.030435200532277423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,768,0.024307199319203696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,51200,0.7685183842976888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,51200,1.0647690455118815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,256,0.016264533003171287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,65536,1.1540437062581381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,12288,0.24326933224995934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,16384,0.259006929397583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,12288,0.21946773529052735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,128,0.0142794668674469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,10240,0.2026378631591797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,10240,0.1737397352854411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,8192,0.1643829345703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,8192,0.13484800656636556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,7168,0.14534719785054523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,7168,0.1162442684173584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,7168,512,0.02067520022392273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,16384,0.32498559951782224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,6144,0.1253002643585205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,6144,0.10187520186106365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,5120,0.10549759864807129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,5120,0.08915627002716064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,5120,0.09195840358734131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,4096,0.08578453063964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,4096,0.07539093494415283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,4096,0.0798858642578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,3584,0.07594026724497477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,3584,0.06815893650054931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,51200,0.8900810877482096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,3072,0.07643413543701172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,16384,0.2838368097941081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,3072,0.0729856014251709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,12288,0.2050293286641439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,3072,0.05641066630681356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,2560,0.06156906684239706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,2560,0.05320320129394531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,10240,0.19121066729227704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,2048,0.051186132431030276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,2048,0.04552426735560099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,1536,0.035753599802653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,1536,0.03784533341725667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,7168,0.13490880330403646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,1024,0.024652800957361855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,1024,0.031275733311971025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,6144,0.10699199835459391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,768,0.02001706759134928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,768,0.028461867570877077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,2560,0.05083306630452474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,512,0.015365333358446757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,512,0.02560639977455139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,8192,0.1437877337137858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,256,0.011752532919247945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,256,0.024182399113972984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,256,0.01567146678765615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,128,0.009918933113416035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,128,0.022357332706451415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,3584,0.0664959987004598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,64,0.009581866860389709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,64,0.02253226637840271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,6144,32,0.01002239982287089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,6144,32,0.022747733195622764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,2048,0.04249173402786255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,1536,0.03370453516642253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,65536,1.2332480112711588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,1024,0.027244800329208375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,65536,0.9094645182291666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,51200,0.8811274846394858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,768,0.02257066567738851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,16384,0.29252694447835287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,51200,0.7229002634684245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,16384,0.2667893409729004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,12288,0.2265461285909017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,12288,0.19003520011901856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,10240,0.19631147384643555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,10240,0.1560010592142741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,8192,0.14665919939676922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,128,0.013314132889111837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,8192,0.12423573335011799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,7168,0.12902186711629232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,7168,0.11082346439361572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,6144,512,0.018990933895111084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,6144,0.11078613599141438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,6144,0.09664639631907145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,12288,0.19802346229553222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,5120,0.09800960222880045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,5120,0.08491199811299642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,65536,1.0505002975463866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,4096,0.08484266599019369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,4096,0.07903786500295004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,16384,0.24714773495992026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,3584,0.06941973368326823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,3584,0.06408533255259195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,10240,0.17204373677571613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,3072,0.060350934664408364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,6144,0.10095360279083251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,3072,0.05728533267974854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,2560,0.0514847993850708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,2560,0.04953386783599854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,7168,0.1180181344350179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,2048,0.04104640086491902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,51200,0.8522474924723307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,2048,0.04169173240661621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,1536,0.03157973289489746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,1536,0.035257601737976076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,8192,0.13005653222401936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,1024,0.022765866915384927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,1024,0.0305184006690979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,3072,0.0512234648068746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,768,0.01845653255780538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,768,0.02866453329722087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,1536,0.030195200443267824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,512,0.014518400033315023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,512,0.025641600290934246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,4096,0.06903466383616129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,256,0.011121066411336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,256,0.022002132733662923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,5120,0.08200106620788575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,128,0.008540800213813782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,2560,0.046366933981577554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,128,0.020148267348607383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,64,0.007292800148328145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,64,0.02034026583035787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,5120,32,0.007893333335717519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,5120,32,0.02040106654167175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,3584,0.05894506772359213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,65536,0.8312927881876627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,65536,0.7415541330973308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,2048,0.0378325343132019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,51200,0.6610602696736654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,51200,0.5782613118489583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,1024,0.024758400519688924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,16384,0.21161707242329916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,16384,0.19454719225565592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,12288,0.1579733371734619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,12288,0.15161919593811035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,128,0.01244586706161499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,256,0.013999999562899271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,10240,0.13661866188049315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,10240,0.12311999797821045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,10240,0.13324693044026692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,8192,0.10857386589050293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,8192,0.10037333170572918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,512,0.017001599073410034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,16384,0.19809172948201498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,7168,0.09560426870981852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,51200,0.6311594645182292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,7168,0.08972586790720621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,5120,768,0.020569600661595664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,7168,0.09126933415730795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,6144,0.0880021333694458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,6144,0.07990399996439615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,12288,0.1591957410176595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,5120,0.06982080141703287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,5120,0.06675733725229899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,5120,0.07578773498535156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,4096,0.055617066224416103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,4096,0.05858560005823771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,3584,0.05242453416188558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,4096,0.054756267865498864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,3072,0.04463680187861125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,65536,0.8074954350789388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,3584,0.053642666339874266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,3072,0.04785386721293132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,2560,0.0368618647257487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,2560,0.042019200325012204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,2048,0.03044053316116333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,2048,0.03643519878387451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,2048,0.031293867031733195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,1536,0.023940267165501912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,1536,0.024913066625595094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,1536,0.03158613244692485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,1024,0.017934934298197428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,1024,0.026782933870951337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,768,0.015211733182271323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,768,0.025309866666793822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,512,0.011978666981061299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,512,0.02169813315073649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,8192,0.1043552001317342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,256,0.008557867010434468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,256,0.020006400346755982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,6144,0.07754133542378744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,128,0.0069354668259620665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,128,0.01840533415476481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,3072,0.041587201754252116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,3584,0.04775786797205607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,64,0.006321066617965698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,4096,32,0.006503466765085857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,32,0.018628267447153728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,4096,64,0.01899519960085551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,2560,0.037929598490397134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,65536,0.8144746780395508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,1024,0.020836265881856282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,768,0.016456533471743265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,512,0.013870933651924133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,51200,0.6944469451904297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,16384,0.2000160058339437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,65536,0.7503231684366862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,51200,0.5734261194864909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,12288,0.15070826212565105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,16384,0.21792426109313964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,12288,0.1503424008687337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,256,0.01134933332602183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,10240,0.14489493370056153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,10240,0.12572480042775472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,8192,0.11743360360463459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,8192,0.09803626537322999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,4096,128,0.009954133629798889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,7168,0.09996799627939859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,7168,0.0887221336364746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,6144,0.0814901351928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,6144,0.078875732421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,12288,0.15786986351013182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,5120,0.07557120323181152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,5120,0.0676800012588501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,16384,0.2081535975138346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,65536,0.8330591837565103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,4096,0.05242986679077148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,6144,0.08269866307576498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,4096,0.06327146689097086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,3584,0.046742399533589676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,3584,0.05185706615447998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,3072,0.041885864734649655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,3072,0.044205868244171144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,10240,0.13737173080444337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,2560,0.03760853211085002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,2560,0.038252798716227214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,7168,0.08838613033294677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,2048,0.029845333099365233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,51200,0.6596544265747071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,2048,0.03312853376070658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,1536,0.02209279934565226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,1536,0.028676267464955645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,8192,0.10030612945556641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,1024,0.015723733107248943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,1024,0.023940267165501912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,3072,0.03879573345184326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,768,0.012466133634249369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,768,0.021182932456334434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,1536,0.023204267024993896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,4096,0.052851200103759766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,512,0.009551999966303508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,512,0.020242132743199668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,5120,0.060959998766581217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,2560,0.035916801293691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,256,0.00680213322242101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,256,0.018994132677714028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,128,0.006155733267466227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,128,0.018590933084487914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,3584,0.04366613229115804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,64,0.0059008002281188965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,64,0.01899306575457255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,1024,0.01939093271891276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3584,32,0.006270933151245117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3584,32,0.018270933628082277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,2048,0.028626133998235066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,65536,0.6545045216878255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,65536,0.6392704010009765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,65536,0.6647551854451497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,51200,0.5382591883341472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,51200,0.5028575897216797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,51200,0.5159626642862956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,16384,0.17346453666687012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,16384,0.1600671927134196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,16384,0.17109012603759766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,12288,0.13223466873168946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,10240,0.10392213662465413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,12288,0.1266869306564331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,8192,0.08338666756947835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,8192,0.087773863474528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,8192,0.08814826806386312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,7168,0.07349226474761963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,7168,0.0781599998474121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,12288,0.13552533785502116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,6144,0.0633674661318461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,6144,0.07174399693806967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,10240,0.10755306879679363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,5120,0.0534880002339681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,5120,0.06151466766993204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,128,0.01009493370850881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,4096,0.04374826749165853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,256,0.011174399654070537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,4096,0.05096746683120727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,3584,0.038991999626159665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,4096,0.04637973308563233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,3584,0.04530133406321208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,3584,0.03915093342463176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,3072,0.033877333005269364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,3072,0.040329599380493165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,3072,0.033608531951904295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,512,0.01344000001748403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,2560,0.028893866141637164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,2560,0.03607786496480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,2048,0.024081067244211832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3584,768,0.015843199690183003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,2048,0.03203840057055156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,1536,0.01931519905726115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,1536,0.027958399057388304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,1024,0.014678399761517844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,1024,0.023126399517059325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,2048,0.02564053336779277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,768,0.01209386686484019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,768,0.022168533007303873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,768,0.014030933380126953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,512,0.009684266646703084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,512,0.020526933670043945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,512,0.012032000223795573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,256,0.007362133264541626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,256,0.018703999121983846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,6144,0.06255786816279094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,128,0.0061365331212679545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,10240,0.1088479995727539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,128,0.01735146641731262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,64,0.005402666827042898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,64,0.01752426624298096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,3072,32,0.005571199953556061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,3072,32,0.017467733224232992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,7168,0.07798826694488525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,65536,0.620037333170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,256,0.0100874662399292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,65536,0.6305258433024089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,51200,0.4842197418212891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,65536,0.5990879694620769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,1536,0.020692267020543418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,16384,0.17739733060201007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,16384,0.16684586207071941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,51200,0.49798186620076496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,12288,0.11591253280639649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,12288,0.11742506821950276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,12288,0.12709226608276367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,10240,0.0971946636835734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,10240,0.10600852966308594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,8192,0.08023146788279215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,8192,0.08743360042572021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,8192,0.07726720174153646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,7168,0.07028266588846842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,7168,0.07920533021291097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,2560,0.030792532364527385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,6144,0.061860267321268717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,5120,0.05448106527328491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,6144,0.07045973141988118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,6144,0.05758506854375204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,1024,0.016596266627311708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,5120,0.05167359908421835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,5120,0.060039468606313075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,4096,0.0425866683324178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,4096,0.04950613180796305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,3072,128,0.009036800265312195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,16384,0.1437983989715576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,51200,0.48248958587646484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,3584,0.03778133392333984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,3072,0.03288639982541402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,3584,0.04451839923858643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,3584,0.03601706822713216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,3072,0.04002453486124675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,2560,0.03602453470230103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,10240,0.10068586667378743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,2048,0.023923200368881226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,2048,0.03221120039621989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,7168,0.06809279918670655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,1536,0.019500799973805747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,1536,0.02839039961496989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,1536,0.019126399358113607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,1024,0.012710400422414145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,1024,0.023639466365178427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,768,0.00992746651172638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,768,0.02061226765314738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,2560,0.02938026587168376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,512,0.007693866888682048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,512,0.019129600127538046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,5120,0.0492576003074646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,256,0.006062933305899302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,256,0.017767467101415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,4096,0.04074346621831258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,128,0.0053727999329566956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,128,0.01851093371709188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,128,0.008436266581217449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,64,0.00557226687669754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,64,0.017913599809010826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2560,32,0.0056991999348004665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2560,32,0.017754666010538735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,1024,0.015416533748308817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,65536,0.4220117251078288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,3072,0.030723200241724653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,65536,0.5278421401977539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,2560,0.02763306697209676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,51200,0.3265642801920573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,2048,0.023355732361475624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,16384,0.10628693103790283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,51200,0.4166271845499675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,16384,0.13882239659627277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,12288,0.08285333315531412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,12288,0.1063914696375529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,10240,0.06757226785024008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,10240,0.09016319910685221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,768,0.013142399986584983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,8192,0.05503146648406983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,8192,0.07378559907277425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,512,0.011464533209800721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,7168,0.04885013500849406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2560,256,0.009342933694521587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,7168,0.06701013247172037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,6144,0.04235519965489705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,6144,0.058815999825795495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,12288,0.10864960352579753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,5120,0.036322132746378584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,5120,0.04992853403091431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,65536,0.5771423975626628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,4096,0.029402667284011842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,4096,0.04137493371963501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,10240,0.09683093229929605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,3584,0.025468800465265912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,6144,0.053257600466410315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,3584,0.03696426550547282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,3072,0.022386133670806885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,3072,0.03445119857788086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,51200,0.4538901329040527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,2560,0.01998186707496643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,2560,0.031124265988667806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,16384,0.14535679817199706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,2048,0.016405333081881204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,7168,0.0610975980758667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,2048,0.027293866872787474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,1536,0.013435733318328858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,1536,0.02369173367818197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,8192,0.06961920261383056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,1024,0.010099200407663982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,1024,0.02111999988555908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,3072,0.02755733331044515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,768,0.008540800213813782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,768,0.020140800873438516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,1536,0.016834133863449098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,512,0.007039999961853028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,5120,0.045645864804585774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,512,0.019117865959803262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,256,0.005686399837334951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,256,0.017574399709701538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,2560,0.025514666239420575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,128,0.004922666649023692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,3584,0.031753599643707275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,128,0.016982400417327882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,4096,0.03626879851023356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,64,0.004484266539414724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,1024,0.013661866386731466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,64,0.01687039931615194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,2048,32,0.00462719996770223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,2048,0.020694400866826376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,2048,32,0.01675093372662862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,65536,0.33373867670694984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,65536,0.47853867212931317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,51200,0.2665482680002848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,51200,0.37826665242513025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,16384,0.08510826428731283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,16384,0.12809173266092938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,16384,0.10510719617207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,12288,0.06404480139414469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,12288,0.09731626510620117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,10240,0.05469119946161906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,10240,0.08410133520762125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,65536,0.38186025619506836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,8192,0.04440960089365641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,256,0.008241066833337148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,8192,0.06814506848653158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,7168,0.039214932918548585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,7168,0.060857601960500086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,7168,0.047416532039642335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,6144,0.034016001224517825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,51200,0.3095936139424642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,6144,0.052541867891947425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,512,0.009859200318654377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,5120,0.028750934203465778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,5120,0.044948267936706546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,768,0.011492266257603963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,4096,0.023804799715677897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,4096,0.03707093397776286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,4096,0.029572266340255737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,2048,128,0.007492266595363617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,3584,0.020786132415135702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,3072,0.018518400192260743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,3584,0.02585066755612691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,3584,0.03449066480000813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,2560,0.016476800044377647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,3072,0.031202133496602374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,2560,0.02055786649386088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,2560,0.028757333755493164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,2048,0.013742933670679728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,12288,0.08020479679107666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,2048,0.02456000049908956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,1536,0.011566932996114094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,1536,0.022465066115061442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,1024,0.00876693328221639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,1024,0.020478934049606323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,1024,0.011473066608111064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,768,0.007518933216730754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,768,0.01970133384068807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,10240,0.06711146831512452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,512,0.006229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,512,0.01824000080426534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,512,0.008834133545557659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,256,0.005054933329423269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,256,0.017338667313257852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,256,0.0075882668296496075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,128,0.004478933413823446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,128,0.016574933131535848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,8192,0.054756267865498864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,64,0.004285866518815359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,64,0.01641279955705007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1536,32,0.004353066782156626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1536,32,0.01609386702378591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,6144,0.04231573343276977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,5120,0.03492693503697713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,65536,0.21956480344136559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,51200,0.17484159469604493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,3072,0.02295680046081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,65536,0.43153174718221027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,16384,0.05804160038630167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,2048,0.017543466885884602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,1536,0.014293332894643148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,51200,0.3443189303080241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,16384,0.11907626787821453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,16384,0.09272747039794922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,12288,0.05599679946899414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,10240,0.04524693489074707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,12288,0.07015999952952066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,12288,0.09383253256479898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,8192,0.030583467086156207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,10240,0.0802186648050944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,8192,0.06290880044301352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,768,0.010039466619491576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,7168,0.05450453360875448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,6144,0.027194666862487792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,6144,0.04701013167699178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1536,128,0.006896000107129414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,7168,0.029982932408650714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,5120,0.022689066330591836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,5120,0.04006719986597697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,4096,0.019170133272806804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,4096,0.03402239878972371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,65536,0.3503125190734863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,51200,0.2774709383646647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,3584,0.017127466201782227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,3584,0.03105386694272359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,10240,0.05986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,3072,0.01461013356844584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,8192,0.04837226470311483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,3072,0.028805333375930785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,7168,0.042115199565887454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,3072,0.019825067122777304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,2560,0.013298133015632629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,2048,0.011386666695276897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,2560,0.02527466615041097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,2048,0.02326186696688334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,2048,0.014307199915250143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,1536,0.008389332890510559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,6144,0.03641386826833089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,1024,0.006705066561698914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,1024,0.019054933389027914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,1024,0.009998933474222819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,768,0.005987200140953064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,768,0.018649599949518838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,5120,0.031049599250157673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,512,0.005132799843947092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,512,0.01725333333015442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,1536,0.020899200439453126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,4096,0.025731199979782106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,256,0.004328533510367076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,256,0.01687893271446228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,128,0.003886933376391729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,128,0.01583253343900045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,128,0.00581226646900177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,64,0.003634133438269297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,64,0.015672533710797628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,1024,32,0.003787733366092046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,1024,32,0.015795200069745382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,65536,0.17737812995910646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,3584,0.02300800085067749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,51200,0.1423818588256836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,2560,0.017514665921529136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,1536,0.01195199986298879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,16384,0.046625065803527835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,51200,0.2196213404337565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,16384,0.11098453203837078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,51200,0.32259947458902993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,12288,0.03628373146057129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,768,0.008691199620564779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,12288,0.08588693141937256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,10240,0.03092693289120992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,10240,0.07164693673451741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,8192,0.02534079949061076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,8192,0.056908798217773435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,65536,0.4105077425638835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,7168,0.02113599975903829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,7168,0.04833920001983642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,512,0.00782293329636256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,1024,256,0.006421333551406861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,6144,0.01884053349494934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,6144,0.04240106741587321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,5120,0.016421332955360413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,5120,0.03630826473236084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,5120,0.026703999439875288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,4096,0.013945600390434265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,4096,0.03086506724357605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,65536,0.2752789179484049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,3584,0.012333866953849793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,3584,0.028039467334747315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,16384,0.07376853624979654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,12288,0.05708266496658325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,3072,0.01123199959595998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,3072,0.026199465990066527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,10240,0.049429333209991454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,2560,0.009585066636403402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,2560,0.02488213380177816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,2048,0.00844053328037262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,2048,0.022371200720469157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,8192,0.0401472012201945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,1536,0.007187200089295705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,1536,0.02039573391278585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,7168,0.03540586630503337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,1024,0.005869866907596588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,1024,0.018812799453735353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,1024,0.009980799754460652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,768,0.005243733525276184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,768,0.018329600493113198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,6144,0.031082665920257567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,512,0.0046165332198143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,512,0.017017600933710735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,4096,0.022453333934148154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,256,0.0040224000811576845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,256,0.016292267044385276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,3584,0.019959467649459838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,128,0.0036256000399589538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,128,0.01585493286450704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,3072,0.01768853267033895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,2560,0.0160863995552063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,64,0.003416533271471659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,768,32,0.003455999990304311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,64,0.015496533115704855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,2048,0.01385599970817566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,768,32,0.01556373337904612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,65536,0.11829973061879476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,65536,0.3777994791666667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,1536,0.012272000312805176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,51200,0.09583253065745036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,51200,0.30036694208780923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,16384,0.036149334907531736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,16384,0.11019306977589924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,768,0.008921600381533305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,12288,0.03109760085741679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,512,0.008380800485610962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,12288,0.08410879770914713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,10240,0.028819199403127032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,10240,0.06960000197092692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,256,0.00724480003118515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,8192,0.020745599269866945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,8192,0.05429120063781738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,768,128,0.006681600213050842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,7168,0.018281600872675576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,7168,0.046817068258921304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,6144,0.01696746746699015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,6144,0.040538668632507324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,65536,0.30963093439737954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,5120,0.013823999961217245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,5120,0.035018666585286455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,12288,0.061169068018595375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,4096,0.011730133493741354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,51200,0.24421119689941406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,4096,0.030270934104919434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,3584,0.010659199953079224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,3584,0.02967360019683838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,16384,0.07781973679860434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,3072,0.009468799829483033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,3072,0.025963733593622845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,10240,0.05311893224716187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,2560,0.009494400024414063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,2560,0.0249834676583608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,8192,0.04087786674499512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,2048,0.008138666550318401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,2048,0.021964800357818604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,6144,0.03179626663525899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,1536,0.007085866729418437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,7168,0.036525865395863846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,1536,0.019831466674804687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,1024,0.005366399884223938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,1024,0.01829866568247477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,3072,0.017164800564448038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,768,0.0048426667849222815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,768,0.01755946675936381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,4096,0.02177066604296366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,512,0.00420053352912267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,512,0.01700586676597595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,5120,0.026742400725682576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,256,0.003681066632270813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,256,0.015561599532763162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,3584,0.01956160068511963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,128,0.0034101332227389016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,2560,0.015101866920789084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,128,0.01518186628818512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,64,0.0032458665470282235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,64,0.015078399578730264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,512,32,0.0033952000240484873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,512,32,0.01546986699104309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,65536,0.08011946678161622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,2048,0.01260586678981781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,65536,0.3583477338155111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,51200,0.06451946496963501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,1536,0.010829866925875346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,51200,0.20909226735432945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,51200,0.2843818664550781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,1024,0.008693333466847737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,16384,0.02669866681098938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,12288,0.021922133366266885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,16384,0.10297493139902751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,12288,0.07815786997477213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,10240,0.017173333962758382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,10240,0.06395839850107829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,65536,0.26014827092488607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,8192,0.013804800311724343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,8192,0.04871466557184855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,256,0.006017066538333893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,16384,0.06717013518015544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,128,0.00555626650651296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,7168,0.04231573343276977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,6144,0.013327999909718832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,512,0.0071263998746871945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,6144,0.03666986624399821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,512,768,0.007754666606585185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,5120,0.011529599626859028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,10240,0.044809599717458085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,5120,0.031883732477823896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,4096,0.009848533074061076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,4096,0.02727786699930827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,4096,0.01961173415184021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,3584,0.009013332923253377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,3584,0.026409600178400678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,3072,0.008232533435026805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,3072,0.024141865968704223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,7168,0.014780799547831217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,2560,0.00735040009021759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,12288,0.052647467454274496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,2560,0.022949333985646567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,2560,0.013763200243314108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,2048,0.006477866570154827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,2048,0.02149440050125122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,2048,0.011713066697120666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,1536,0.005752533177534739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,1536,0.019744000832239785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,8192,0.03539946476618449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,1024,0.0047658666968345646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,1024,0.0180351992448171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,6144,0.02714346647262573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,7168,0.031378134091695147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,768,0.0042805333932240805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,768,0.017084799210230508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,5120,0.023500800132751465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,512,0.0038143999874591826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,512,0.016566399733225504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,256,0.0034346667428811393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,256,0.015530666708946228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,3584,0.017359999815622966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,128,0.003134933362404505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,128,0.015019733707110086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,3072,0.015548800428708395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,64,0.0030261332790056865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,64,0.01471573313077291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,256,32,0.0030421334008375804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,256,32,0.014827733238538107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,65536,0.07161706288655599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,65536,0.3506752014160156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,65536,0.2322528044382731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,1536,0.010024533669153849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,51200,0.05803413391113281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,51200,0.18306345939636232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,51200,0.27584854761759436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,16384,0.021912533044815063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,16384,0.0625279982884725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,12288,0.0123690664768219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,16384,0.09968852996826172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,12288,0.07575146357218424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,10240,0.011175466577212016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,10240,0.060989868640899655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,8192,0.01134399970372518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,8192,0.04610133171081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,1024,0.008172800143559773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,7168,0.010829866925875346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,7168,0.04003413518269856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,768,0.007247999807198842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,6144,0.011431466539700825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,6144,0.035327998797098796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,512,0.0066442668437957765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,5120,0.00997226635615031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,5120,0.030507733424504597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,256,0.005701333284378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,4096,0.008584533135096233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,256,128,0.005167999863624572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,4096,0.027539199590682982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,4096,0.018484266599019368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,3584,0.007916800181070964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,3584,0.017084799210230508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,3584,0.02598080039024353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,3072,0.007235200206438701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,3072,0.01493013302485148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,3072,0.024526933828989662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,2560,0.00644053320089976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,2560,0.02215786576271057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,2048,0.005766400198141734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,2048,0.02076373298962911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,12288,0.048547200361887616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,1536,0.00499839981396993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,10240,0.04160319964090983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,1536,0.01880426605542501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,1536,0.010328533252080281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,1024,0.004331733286380768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,8192,0.033029333750406904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,1024,0.018092799186706542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,768,0.003992533435424169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,768,0.017018665870030723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,512,0.0036159999668598174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,512,0.016432000199953715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,7168,0.028985599676767986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,256,0.0033130665620168054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,256,0.01539306640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,6144,0.025252266724904375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,256,0.006126933296521505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,128,0.00306986669699351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,128,0.01467519998550415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,64,0.0029205332199732465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,64,0.014745600024859109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,128,32,0.0028864001234372456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,5120,0.021689599752426146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,128,32,0.014605866869290671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,65536,0.06616106828053793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,51200,0.0525984009106954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,65536,0.3458101272583008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,2560,0.01350826621055603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,51200,0.2735605239868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,12288,0.010780800382296245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,2048,0.011681066950162251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,16384,0.09651626745859782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,10240,0.009505066275596618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,12288,0.07209386825561523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,10240,0.05880853335062662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,8192,0.009914666414260864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,7168,0.009315199653307597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,8192,0.04423679908116658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,7168,0.03886933326721191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,6144,0.008821333448092144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,5120,0.007978666822115581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,6144,0.03275306622187297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,4096,0.007176533341407776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,5120,0.030497066179911297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,3584,0.006931200126806895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,4096,0.026972800493240356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,3072,0.006523733337720235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,3584,0.025493333737055462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,2560,0.006182399888833364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,3072,0.023512534300486245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,2048,0.005464533468087515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,2560,0.02214933236440023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,1536,0.004795733094215393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,2048,0.020040533939997354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,1024,0.0040287998815377556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,1024,0.017403733730316163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,768,0.0037077332536379496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,768,0.017035732666651406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,512,0.003385599950949351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,512,0.016218666235605875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,16384,0.020589866240819297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,256,0.0030634666482607523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,128,0.0028213332096735638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,1024,0.008461866776148479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,256,0.015196800231933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,128,0.014666666587193808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,768,0.007463466624418895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,32,0.002771199991305669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,64,64,0.002701866626739502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,64,0.014845866958300272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,32,0.014801067113876343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,65536,0.06556373437245687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,51200,0.0525984009106954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,16384,0.01849173307418823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,12288,0.009971200426419576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,512,0.0070933332045873005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,16384,0.09463040033976236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,10240,0.008886399865150451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,12288,0.07117653687795003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,51200,0.27266454696655273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,10240,0.05646186669667562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,7168,0.007236266632874806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,8192,0.042896000544230144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,6144,0.006860800087451935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,7168,0.03719040155410767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,5120,0.00659093310435613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,6144,0.032451200485229495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,4096,0.006145066519578298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,5120,0.030643200874328612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,4096,0.026769065856933595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,3072,0.00621013343334198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,3584,0.025527467330296833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1280,128,128,0.005593599875768026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,2560,0.006205866734186808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,3072,0.023254400491714476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,2048,0.005549866457780202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,1536,0.004844800134499868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,2048,0.02041813333829244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,64,1536,0.01995413303375244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,1024,0.0040554667512575785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,1536,0.019288533926010133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,768,0.003753600021203359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,1024,0.017760000626246133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,512,0.0034783999125162757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,768,0.016757333278656007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,256,0.003139200061559677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,512,0.01651093363761902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,128,0.0028938665986061097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,256,0.015355733036994935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,64,0.0027253332237402597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,128,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,32,0.0027306665976842242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,64,0.014685866236686707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,32,0.014616533120473226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,8192,0.007692799965540569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1280,32,3584,0.006586666901906331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,2560,0.0223306675752004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1280,32,65536,0.34534934361775715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,16384,1.456653849283854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,12288,1.181161626180013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,12288,2.291198984781901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,16384,3.075989278157552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,16384,2.4906219482421874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,12288,1.6849620819091797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,10240,0.9810111999511719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,8192,1.5492938995361327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,10240,1.402451197306315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,10240,1.9511082967122395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,8192,0.7665920257568359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,51200,5.0455774943033855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,8192,1.098849105834961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,7168,0.6699551900227865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,7168,1.2860404968261718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,6144,0.6017226537068685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,7168,0.9621685028076172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,6144,1.1089152018229167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,6144,0.8380906422932943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,5120,0.9438997268676758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,5120,0.5014399846394857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,5120,0.6939605077107747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,4096,0.7857290903727214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,4096,0.4330037434895833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,4096,0.5837706883748373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,3584,0.6822101593017578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,3584,0.38329172134399414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,3584,0.4960757255554199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,3072,0.5552768071492513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,3072,0.318452262878418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,2560,0.47320960362752273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,3072,0.45485334396362304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,2560,0.2617578665415446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,2560,0.382152525583903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,2048,0.36282774607340496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,2048,0.23303893407185874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,1536,0.27339626948038737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,2048,0.33046080271402994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,1536,0.1786239941914876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,1024,0.19771626790364583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,1024,0.14124800364176432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,768,0.15252265930175782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,1024,0.20239787101745604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,1536,0.2768629391988119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,512,0.11208960215250652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,768,0.11703466574350993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,512,0.09153066476186117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,768,0.17584959665934247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,256,0.0739466667175293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,512,0.15962665875752766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,256,0.08009066581726074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,128,0.06495039860407512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,256,0.1198463996251424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,128,0.07162026564280191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,64,0.05846933523813883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,51200,9.929990641276042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,64,0.0717525323232015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,65536,128,0.1137727975845337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,32,0.06512746810913086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,65536,32,0.07104427019755045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,65536,51200,9.243628946940104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,16384,1.3807231903076171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,16384,2.720197296142578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,16384,1.758460744222005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,12288,2.052881113688151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,51200,4.537466430664063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,12288,1.0159189224243164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,12288,1.3037002563476563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,65536,5.90240224202474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,10240,1.7032853444417317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,10240,0.857912508646647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,51200,7.489036560058594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,8192,0.691050656636556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,10240,1.0813109079996743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,8192,0.8801375706990561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,8192,1.2574026743570963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,51200,8.319492085774739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,7168,0.60862611134847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,7168,1.1445877075195312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,7168,0.776093864440918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,6144,0.5369802474975586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,6144,0.986566416422526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,6144,0.6468330383300781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,5120,0.45769812266031906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,5120,0.8282911936442057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,4096,0.6395349502563477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,4096,0.3886890729268392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,5120,0.5556906382242839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,3584,0.3496586799621582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,3072,0.2936842600504557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,3584,0.5861888249715169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,3584,0.40006507237752276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,3072,0.4971274693806966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,2560,0.2612031936645508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,3072,0.34357118606567383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,2560,0.291543451944987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,2560,0.40736961364746094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,2048,0.3546325365702311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,65536,10.200620524088542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,2048,0.212881072362264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,1536,0.16744213104248046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,65536,10.72693583170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,1536,0.2606719970703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,2048,0.2518346627553304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,1536,0.22434666951497398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,768,0.10719679991404216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,512,0.10785706837972005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,512,0.08562560081481933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,1024,0.1546677271525065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,768,0.13577920595804852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,768,0.1508672078450521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,1024,0.18328746159871417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,512,0.11543573538462322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,256,0.070632537206014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,256,0.09413973490397134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,4096,0.4521546681722005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,256,0.07086293697357178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,64,0.049876264731089276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,128,0.06347839832305908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,32,0.04743680159250895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,51200,128,0.0902079979578654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,64,0.06265706618626912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,32,0.06323626836140951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,51200,1024,0.1251360019048055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,51200,128,0.06532586812973022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,51200,1.3948874155680338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,65536,1.795244852701823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,16384,0.7205535888671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,16384,0.49956480662027997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,51200,2.445216115315755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,12288,0.3395274798075358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,12288,0.5877536137898762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,16384,0.5583338419596354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,65536,2.7184511820475263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,51200,1.8994752248128255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,12288,0.4188543955485026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,10240,0.3064319928487142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,10240,0.3589834531148275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,8192,0.40206187566121415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,8192,0.2888117472330729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,8192,0.27296746571858727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,65536,3.637650044759115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,7168,0.2105621337890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,7168,0.32046292622884115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,6144,0.18639146486918132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,6144,0.27932694753011067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,7168,0.2572949409484863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,5120,0.2265706698099772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,5120,0.155457067489624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,6144,0.2277344067891439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,5120,0.1884160041809082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,4096,0.18846186002095539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,4096,0.13610560099283855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,4096,0.15425705909729004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,3584,0.1619274616241455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,3584,0.11619626681009929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,3072,0.1434655984242757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,3584,0.1421130657196045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,3072,0.10530773003896077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,2560,0.11864426930745442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,2560,0.10232960383097331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,2048,0.10102399984995525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,1536,0.07629120349884033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,2048,0.07653546333312988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,2048,0.08596479892730713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,1536,0.06222293376922607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,1024,0.055567999680837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,1024,0.051671465237935386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,768,0.04377919832865397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,1536,0.07183893521626791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,512,0.03340693314870198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,768,0.044622933864593504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,512,0.039289601643880206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,512,0.040779733657836915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,256,0.02361066738764445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,128,0.018278400103251137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,10240,0.450161075592041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,128,0.029688533147176104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,64,0.017271467049916587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,128,0.03210560083389282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,64,0.030773333708445233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,32,0.017106133699417114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,32,0.03033493359883626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,2560,0.0860864003499349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,3072,0.12562452952067057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,16384,1024,0.05555200179417928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,768,0.050273064772288004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,16384,256,0.03325013319651286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,16384,0.5269376118977864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,65536,2.392211151123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,51200,1.7333386739095051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,16384,256,0.0344703992207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,16384,0.34834880828857423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,65536,1.9033685048421223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,12288,0.402286942799886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,12288,0.2588650703430176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,16384,0.43855145772298176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,12288,0.32275091807047523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,65536,1.3771007537841797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,10240,0.37458346684773763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,10240,0.22554987271626792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,10240,0.2741621335347494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,8192,0.27759361267089844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,8192,0.19857813517252604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,7168,0.1623136043548584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,51200,1.0297375996907552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,7168,0.2015519936879476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,8192,0.22481172879536948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,6144,0.14767146110534668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,6144,0.21022399266560873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,5120,0.12370453675587971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,5120,0.16872426668802898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,4096,0.14203200340270997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,4096,0.10656747023264568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,4096,0.12116693655649821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,3584,0.12071786721547444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,3584,0.09684906800587972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,3584,0.10982720057169597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,3072,0.10474773248036702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,3072,0.08650453090667724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,3072,0.09427946408589681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,51200,1.534228261311849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,2560,0.08920000394185384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,2560,0.07276586691538492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,2560,0.08171626726786295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,2048,0.0733354647954305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,2048,0.06162559986114502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,2048,0.06766293048858643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,1536,0.05826880137125651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,1536,0.053174400329589845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,1536,0.05524479945500692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,1024,0.04135253429412842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,1024,0.0431658665339152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,768,0.03434559901555379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,1024,0.04461866617202759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,768,0.03801279862721761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,768,0.03909226655960083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,512,0.026424533128738402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,512,0.033019733428955075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,256,0.01879253387451172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,512,0.03265066742897034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,256,0.02792746623357137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,128,0.025476266940434773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,128,0.025156267484029132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,64,0.012552533547083536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,64,0.025834665695826216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,32,0.013230933745702108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,7168,0.2581450621287028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,12288,32,0.026317866643269856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,6144,0.17196906407674153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,5120,0.14627092679341633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,12288,128,0.014406399925549826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,51200,1.2795647939046224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,65536,1.6516415913899738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,51200,2.1183595021565753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,16384,0.6857664108276367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,16384,0.42544959386189773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,12288,256,0.027674667040507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,65536,2.7679840087890626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,12288,0.3190261205037435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,16384,0.48391040166219074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,12288,0.5007594744364421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,10240,0.4055541356404622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,10240,0.26898345947265623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,8192,0.2081962744394938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,8192,0.3456384023030599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,8192,0.2596810658772787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,7168,0.18911892573038738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,7168,0.2983637491861979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,6144,0.24676159222920738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,6144,0.1640874703725179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,5120,0.2133258660634359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,5120,0.14103573163350422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,5120,0.162828795115153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,4096,0.17080106735229492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,4096,0.11531306902567547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,3584,0.14938027064005535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,3584,0.10485119819641113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,65536,2.217535909016927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,3072,0.1375402609507243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,12288,0.3579594612121582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,3072,0.09794666767120361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,51200,1.698094940185547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,10240,0.31289707819620766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,2560,0.11361599763234456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,2560,0.08636480172475179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,2048,0.09304959774017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,2048,0.06990933418273926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,1536,0.07476692994435628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,7168,0.23032746315002442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,1536,0.05593066612879435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,1024,0.05073493321736654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,1024,0.04377706845601399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,6144,0.1952799956003825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,768,0.04090880155563355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,768,0.03731413284937541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,4096,0.13405653635660808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,768,0.029730133215586346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,512,0.030521599451700847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,3584,0.11297813256581624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,256,0.01686720053354899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,256,0.024425599972407022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,512,0.029084799687067668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,128,0.011729066570599873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,128,0.02250986695289612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,3072,0.09907626310984294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,64,0.00962773362795512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,64,0.022694400946299233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,2560,0.0836885372797648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,10240,32,0.010166399677594503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,10240,32,0.023035732905069987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,2048,0.06386666695276896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,1536,0.04914666811625163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,1024,0.03800959984461467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,65536,0.9918389638264975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,512,0.024396799008051553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,51200,0.7528106689453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,65536,1.5387840270996094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,256,0.020297600825627645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,16384,0.3562517484029134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,16384,0.26980692545572915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,65536,1.677994664510091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,12288,0.2690090815226237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,12288,0.20431253115336098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,12288,0.2942591985066732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,10240,0.22577385902404784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,51200,1.290229288736979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,10240,128,0.01725546717643738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,8192,0.18928853670756024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,51200,1.0803733825683595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,7168,0.11652373472849528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,7168,0.19370452562967938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,6144,0.13423040707906086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,6144,0.10069866975148518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,10240,0.16400532722473143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,6144,0.16481599807739258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,5120,0.1128981351852417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,5120,0.08832746346791585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,5120,0.13548800150553386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,4096,0.09135146935780844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,7168,0.16683200200398762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,8192,0.1448202610015869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,4096,0.11043519973754883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,3584,0.08016853332519532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,3584,0.06777386665344239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,3584,0.09609386920928956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,3072,0.06078506708145141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,3072,0.08296319643656412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,2560,0.05953280131022135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,3072,0.07090240319569906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,2560,0.0538645346959432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,2560,0.06869866847991943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,2048,0.046987732251485184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,1536,0.03813973267873128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,2048,0.05385706822077433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,1024,0.026881066958109538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,1536,0.03902293443679809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,1024,0.03240320086479187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,1024,0.03181653420130412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,768,0.022188800573349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,768,0.02866986592610677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,512,0.016960000991821288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,512,0.025737599531809492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,16384,0.4002442677815755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,256,0.012489599982897441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,256,0.02257919907569885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,4096,0.07430933316548666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,128,0.009122133255004883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,128,0.021259733041127524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,10240,0.25785279273986816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,2048,0.04881706635157267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,64,0.007855999966462452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,8192,0.21705172856648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,8192,32,0.00817386656999588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,64,0.021600000063578286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,8192,32,0.021513599157333373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,65536,0.9013802846272787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,65536,1.035812250773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,65536,1.2916895548502603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,1536,0.0426090677579244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,768,0.02604159911473592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,51200,0.7132533391316731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,16384,0.3216362635294596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,51200,1.0541930516560873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,512,0.021474132935206093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,16384,0.2362816015879313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,51200,0.8232671737670898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,16384,0.26373546918233237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,12288,0.24692692756652831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,256,0.017704532543818156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,12288,0.19942827224731446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,12288,0.19791146914164226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,10240,0.2063221295674642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,8192,0.17118293444315594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,8192,0.11989759604136149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,10240,0.18133974075317383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,8192,0.13785173098246256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,7168,0.14495894114176433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,6144,0.1217845360438029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,6144,0.1022602637608846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,6144,0.10662293434143066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,5120,0.10581653118133545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,5120,0.08214399814605713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,4096,0.08396053314208984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,8192,128,0.014987732966740927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,4096,0.07453227043151855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,3584,0.07445973555246989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,4096,0.07478400071461996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,3584,0.06372586488723755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,3584,0.06934399604797363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,3072,0.06461333433787028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,3072,0.056536531448364256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,3072,0.05808639923731486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,2560,0.057410132884979245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,2560,0.049721598625183105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,2048,0.045577601591746016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,2560,0.0504309336344401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,2048,0.04463893175125122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,1536,0.03573226531346639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,2048,0.04288640022277832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,1536,0.03769173224767049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,1536,0.03533226648966471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,1024,0.031197865804036457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,1024,0.028343466917673747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,768,0.021272534132003786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,768,0.027185066541035967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,768,0.02601813276608785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,512,0.01586666703224182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,512,0.026692267258961993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,512,0.02178666591644287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,256,0.011429333686828613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,256,0.021642667055130006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,256,0.01814613342285156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,128,0.009141332904497783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,128,0.02304960091908773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,128,0.016561067104339598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,10240,0.16803520520528156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,64,0.010018133123715718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,64,0.02209386626879374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,32,0.00967680017153422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,32,0.022957867383956908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,7168,7168,0.11334293683369953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,7168,0.12758506933848063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,7168,5120,0.09426773389180501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,7168,1024,0.025101866324742633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,65536,0.8539285024007162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,16384,0.3099285443623861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,51200,0.9826762517293295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,16384,0.230948273340861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,65536,1.2319807688395181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,12288,0.17009280522664388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,12288,0.24509013493855797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,10240,0.19870932896931964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,10240,0.15036800702412922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,8192,0.15995200475056964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,10240,0.2049013296763102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,8192,0.12055679957071941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,7168,0.13984106381734213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,51200,0.6779274622599284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,7168,0.10794026851654052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,6144,0.12008639971415203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,6144,0.09265920321146646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,5120,0.09945387045542399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,5120,0.07986773649851481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,65536,1.3205589294433593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,51200,0.9988032023111979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,16384,0.315229860941569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,12288,0.23360533714294435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,4096,0.08225813706715902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,3584,0.07116266886393229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,4096,0.07041066487630208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,3584,0.06413439909617105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,3072,0.06414719820022582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,3072,0.05787306626637777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,2560,0.05414400100708008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,2560,0.04949653148651123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,8192,0.16936960220336914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,2048,0.042878933747609455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,2048,0.041442131996154784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,7168,0.15103573799133302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,1536,0.03369813362757365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,1536,0.03407253424326579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,5120,0.10633920033772785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,1024,0.023468800385793052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,1024,0.029044266541798907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,1024,0.025563732782999678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,768,0.01839146614074707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,768,0.025518933931986492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,6144,0.12899200121561688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,512,0.014036267002423605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,512,0.022833067178726196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,4096,0.08490453561147054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,256,0.00972160001595815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,3584,0.07387946446736654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,256,0.02068480054537455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,256,0.01334826648235321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,128,0.007289599875609081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,128,0.019082667430241902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,64,0.0065194666385650635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,3072,0.06331946849822997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,6144,32,0.006888533135255177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,64,0.019230933984120686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,2560,0.0531765341758728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,6144,32,0.019336533546447755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,2048,0.042631467183430985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,1536,0.032739200194676715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,65536,0.9116362889607748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,65536,1.0309418360392253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,51200,1.0585503896077475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,768,0.02010986606280009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,65536,1.3337003072102864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,51200,0.7172042846679687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,16384,0.23461653391520182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,16384,0.33281065622965494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,12288,0.24592320124308267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,16384,0.25509546597798666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,12288,0.17179946899414061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,12288,0.19258240063985188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,10240,0.20373652776082357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,10240,0.14181866645812988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,51200,0.7983690897623699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,8192,0.12085226376851399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,8192,0.1342741330464681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,10240,0.16637439727783204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,512,0.016617600123087564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,7168,0.14653226534525554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,6144,0.11973760128021241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,7168,0.10981973012288411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,6144,0.09456533590952555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,5120,0.10204479694366456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,5120,0.08364266554514567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,4096,0.08243839740753174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,4096,0.07041813532511393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,3584,0.08004373709360758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,3584,0.06425173282623291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,3584,0.05972479979197184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,3072,0.0654368003209432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,6144,128,0.011801600456237793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,3072,0.05595733324686686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,3072,0.051844267050425205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,2560,0.056447998682657874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,2560,0.04735039869944255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,2048,0.04458560148874919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,2048,0.03999786774317424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,8192,0.16714986165364581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,1536,0.034354134400685625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,1536,0.03425813515981038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,1024,0.0234826664129893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,1024,0.02850240071614583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,7168,0.12069226900736492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,6144,0.1051520029703776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,768,0.01877440015474955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,768,0.02587626576423645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,768,0.018499199549357095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,5120,0.08227519989013672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,512,0.014524799585342408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,512,0.022361600399017335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,256,0.009475200374921163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,512,0.014963199694951376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,256,0.020197333892186482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,256,0.012425600488980611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,128,0.018821332852045694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,64,0.006479999919732411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,64,0.019170133272806804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,32,0.007109333574771881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,5120,32,0.019022933642069497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,2560,0.04382079839706421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,2048,0.03511039813359578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,65536,0.6976469039916993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,1536,0.02717439929644267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,4096,0.0697813351949056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,5120,128,0.007342933118343354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,65536,0.5991903940836589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,16384,0.1770293394724528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,51200,0.467577584584554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,51200,0.556929079691569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,16384,0.16352640787760417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,12288,0.12957759698232013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,12288,0.1364906628926595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,10240,0.10623466968536377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,1024,0.021849600474039714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,8192,0.08816640377044678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,8192,0.08343253135681153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,5120,128,0.010918399691581726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,7168,0.07876693407694499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,10240,0.1086079994837443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,7168,0.07561919689178467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,6144,0.0679701328277588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,6144,0.06765440305074057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,12288,0.17534720102945964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,5120,0.056659201780954994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,5120,0.059404798348744715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,16384,0.2343477408091227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,4096,0.047172268231709794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,4096,0.05040106773376465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,10240,0.15444053014119466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,3584,0.041016534964243574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,6144,0.0948256015777588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,3584,0.0448202649752299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,3072,0.03573866685231526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,3072,0.0403711994489034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,51200,0.7342304229736328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,2560,0.032866134246190386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,65536,0.9352352142333984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,2560,0.03676799933115642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,2048,0.02598506609598796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,7168,0.11056746641794842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,2048,0.03277440071105957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,1536,0.020915200312932335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,1536,0.028861866394678755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,8192,0.1264789342880249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,1024,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,1024,0.025036799907684325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,3072,0.048196268081665036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,768,0.012658133109410604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,1536,0.025562665859858197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,768,0.022593067089716593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,5120,0.07849600315093994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,512,0.009807999928792317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,512,0.02027946710586548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,2560,0.04034773508707683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,256,0.007406933108965556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,1024,0.02050986687342326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,256,0.018819200992584228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,128,0.006165333092212677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,4096,0.06499199867248535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,3584,0.05563626686731974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,128,0.017770665884017944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,64,0.005508266886075338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,64,0.017811199029286705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,4096,32,0.005701333284378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,4096,32,0.017816533644994102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,2048,0.03276693423589071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,65536,0.6272426605224609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,65536,0.5700949350992839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,51200,0.4903498649597168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,51200,0.44792639414469404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,16384,0.1596511999766032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,16384,0.1501471996307373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,16384,0.14470933278401693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,12288,0.1189461310704549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,51200,0.4619701385498047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,12288,0.1170240004857381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,12288,0.11154026985168457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,10240,0.10964159965515137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,10240,0.09832426706949869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,10240,0.09539519945780436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,8192,0.08925653298695882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,8192,0.08082026640574137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,8192,0.07717866897583008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,7168,0.07869546413421631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,7168,0.06861226558685303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,7168,0.07387733459472656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,6144,0.06858773231506347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,6144,0.060794667402903235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,5120,0.05540693203608195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,5120,0.05384106636047363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,5120,0.057258665561676025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,4096,0.04462399880091349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,4096,0.047288533051808676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,4096,0.0438261349995931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,3584,0.03769173224767049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,3584,0.043170134226481124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,3072,0.03312000036239624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,3072,0.038387199242909745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,3072,0.03430826663970947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,2560,0.028140799204508467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,2560,0.03438400030136109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,2560,0.0295413335164388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,2048,0.02307413419087728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,2048,0.030804266532262165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,2048,0.025856000185012818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,1536,0.018798933426539103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,1536,0.02682773272196452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,1536,0.021612799167633055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,1024,0.014149333039919535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,6144,0.06318186521530152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,1024,0.02330346703529358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,768,0.011993599931399028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,768,0.021305600802103676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,768,0.015829333662986757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,512,0.009291733304659527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,512,0.0199072003364563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,512,0.01365013321240743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,256,0.007055999835332234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,256,0.018582399686177573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,256,0.011460266510645549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,128,0.0058378666639328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,128,0.010006399949391682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,128,0.018782933553059898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,64,0.00603413333495458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,64,0.018293333053588868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,256,0.011124266187349956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3584,32,0.006121600170930227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3584,32,0.018321067094802856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,768,0.01595626672108968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,65536,0.6361781438191731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,65536,0.5460671742757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,4096,512,0.013103999694188437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,3584,0.03857920169830322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,16384,0.1599840005238851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,51200,0.4957578659057617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,51200,0.4344810803731282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,16384,0.13897919654846191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,65536,0.6698954900105794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,12288,0.1231317361195882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,12288,0.10600213209788005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,16384,0.18141973813374837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,10240,0.10497386455535888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,12288,0.12907946904500325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,10240,0.09195626576741536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,8192,0.08041280110677083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,8192,0.07515093485514322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,7168,0.06915093262990316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,7168,0.06849813461303711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,65536,0.5383850733439128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,128,0.010699733098347982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,6144,0.06225493351618448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,51200,0.5296202659606933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,6144,0.0605674664179484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,5120,0.05095359881718954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,10240,0.10919040044148762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,5120,0.05251626571019491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,5120,0.05779413382212321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,4096,0.04013653198877971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,8192,0.08887466589609781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,4096,0.04349013169606526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,3584,0.035846400260925296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,7168,0.07821226914723714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,3072,0.03128213286399841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,3584,0.039553066094716385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,2560,0.026583466927210492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,3072,0.03553280035654704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,2560,0.0321397324403127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,2048,0.022297600905100502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,2048,0.028622933228810626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,1536,0.01741973360379537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,1536,0.02490880091985067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,6144,0.06777066389719645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,1024,0.012719999750455221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,1024,0.021729065974553426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,1024,0.015460266669591268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,768,0.0107914666334788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,3584,0.04038826624552409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,768,0.02030186653137207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,512,0.00817386656999588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,3072,0.0343285322189331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,4096,0.04633920192718506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,512,0.010811733206113179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,512,0.019218132893244425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,256,0.006355200211207073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,128,0.005373866856098175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,256,0.00897279977798462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,256,0.01772159934043884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,64,0.004792533318201701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,128,0.017859200636545815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3584,1024,0.01718719998995463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,3072,32,0.005161599814891815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,2048,0.024889600276947022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,32,0.01780479947725932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,65536,0.5062837282816569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,51200,0.3533482551574707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,1536,0.01998186707496643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,2560,0.029959466060002642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,16384,0.11099733511606853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,3072,64,0.01759786605834961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,51200,0.3882698694864909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,16384,0.12705600261688232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,12288,0.08578666845957438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,12288,0.09894933700561523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,10240,0.0714186668395996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,10240,0.08158400058746337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,65536,0.5228288014729817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,8192,0.05971306562423706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,8192,0.06775039831797282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,768,0.013260799646377563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,7168,0.051643733183542886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,7168,0.0626965324083964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,3072,128,0.007922133306662242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,6144,0.04335999886194865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,6144,0.05518293380737305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,65536,0.6061653137207031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,5120,0.03694080114364624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,5120,0.04942506551742554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,51200,0.4779317220052083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,16384,0.15850240389506023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,4096,0.030794666210810347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,4096,0.04006613492965698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,12288,0.12000532944997151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,3584,0.0281333327293396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,3584,0.03598933219909668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,3072,0.025278933842976886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,10240,0.10290559927622478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,3072,0.03302719990412394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,2560,0.020530132452646892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,2560,0.0303327997525533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,8192,0.0807978630065918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,2048,0.017886932690938315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,2048,0.02720213333765666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,7168,0.07259306907653809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,1536,0.014628266294797262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,1536,0.0240938663482666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,1536,0.018875734011332194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,1024,0.01086186667283376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,6144,0.06311999956766765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,1024,0.021596799294153847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,768,0.008989866574605305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,768,0.019668267170588175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,5120,0.052604798475901285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,512,0.00734506646792094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,512,0.018862932920455933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,4096,0.04256426493326823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,256,0.005890133480230967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,256,0.017362133661905924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,3584,0.0377781351407369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,3072,0.032333866755167646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,128,0.0050784001747767125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,128,0.016613333423932394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,2560,0.028429865837097168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,64,0.004561066627502441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,64,0.016748799880345663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2560,32,0.004886400202910105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2560,32,0.016724266608556113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,2048,0.02303253412246704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,65536,0.39591573079427084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,1024,0.014605866869290671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,768,0.012641066312789917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,65536,0.4691146532694499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,51200,0.30753920873006185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,16384,0.09641813437143962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,51200,0.3547744115193685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,16384,0.11913493474324544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,12288,0.07316693464914957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,12288,0.08906986713409423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,512,0.010575999816258747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,10240,0.0621888001759847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,10240,0.07752426465352377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,256,0.00893440047899882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,8192,0.05056213140487671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2560,128,0.00807360013326009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,8192,0.06553706725438437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,7168,0.044232531388600664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,7168,0.05880959828694662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,6144,0.03889386653900147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,6144,0.05114986499150594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,12288,0.11579093138376872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,5120,0.032501333951950075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,5120,0.04369279940923055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,16384,0.15678292910257977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,65536,0.6134570439656575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,4096,0.027110399802525838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,4096,0.03696000178654989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,3584,0.02402879993120829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,3584,0.03431893189748128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,6144,0.06221226851145426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,3072,0.021042132377624513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,51200,0.4741077423095703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,3072,0.031133866310119628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,2560,0.018243199586868285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,2560,0.02834666570027669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,7168,0.07158186435699462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,2048,0.01818559964497884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,2048,0.02677226662635803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,8192,0.08261333306630453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,1536,0.0150026669104894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,10240,0.10021546681722004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,1536,0.022987733284632363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,1024,0.010339200496673584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,1024,0.02073813279469808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,4096,0.04154986540476481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,768,0.008587732911109924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,768,0.020426666736602782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,3072,0.03243520061175029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,512,0.006984533369541168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,512,0.01889280080795288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,5120,0.05194240013758341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,256,0.005994666616121928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,256,0.017528533935546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,1536,0.017090133825937905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,2560,0.02719680070877075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,128,0.005053866902987162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,128,0.01641279955705007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,3584,0.0362335999806722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,64,0.0044064000248909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,2048,32,0.004474666714668274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,64,0.016306133071581522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,2048,32,0.016245333353678386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,65536,0.27484585444132487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,2048,0.022334933280944824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,65536,0.4302816073099772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,1024,0.013921067118644714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,51200,0.21299734115600585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,16384,0.09169812997182211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,51200,0.3438528060913086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,51200,0.33275092442830406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,16384,0.11257493495941162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,12288,0.06379626592000326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,12288,0.08847359816233316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,10240,0.059741866588592527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,10240,0.07645333607991536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,65536,0.4263466517130534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,8192,0.0445087989171346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,16384,0.11111146608988445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,8192,0.06230400005976359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,7168,0.04109760125478108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,12288,0.08129599889119467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,7168,0.05490026473999023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,7168,0.049710933367411295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,6144,0.03679999907811483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,6144,0.0468938668568929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,128,0.007417599856853485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,10240,0.0681717316309611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,5120,0.029383466641108198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,5120,0.03602773348490397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,5120,0.040415998299916586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,4096,0.02394346594810486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,4096,0.034251733620961504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,3584,0.02073813279469808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,3584,0.03178986708323161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,512,0.009858133395512898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,3072,0.018961066007614137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,768,0.011912533640861511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,3072,0.029206399122873945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,2560,0.01623253325621287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,2560,0.02723199923833211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,2048,256,0.008220799763997396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,2048,0.01402239998181661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,2048,0.02437653342882792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,8192,0.05671679973602295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,1536,0.010803199807802836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,1536,0.02136639952659607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,6144,0.043333331743876144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,1024,0.007896533111731212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,1024,0.0196341335773468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,1024,0.01053653359413147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,768,0.006577066580454509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,3584,0.026680533091227216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,768,0.018422400951385497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,4096,0.03009066581726074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,512,0.005589333176612854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,768,0.009291733304659527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,256,0.004545066754023234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,512,0.017658666769663493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,256,0.016684800386428833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,128,0.00415786678592364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,128,0.015650133291880287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,64,0.003819733361403147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,3072,0.023478400707244874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1536,32,0.004041599979003271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,64,0.017145599921544394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1536,32,0.01577279965082804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,65536,0.18696212768554688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,2560,0.020317866404851278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,65536,0.3540970802307129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,51200,0.14984532992045085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,2048,0.0164490669965744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,16384,0.055061332384745275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,51200,0.28171094258626306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,16384,0.0978869358698527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,12288,0.04092906713485718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,12288,0.07496106624603271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,256,0.006669866542021434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,12288,0.07158079942067465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,10240,0.03750933408737182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,10240,0.06286293268203735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,8192,0.028267733256022137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,8192,0.050777598222096765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,51200,0.2737621307373047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,7168,0.025006934007008867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,7168,0.044812798500061035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,1536,0.013674666484196981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,6144,0.022395733992258707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,7168,0.04350933233896891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,6144,0.03729493220647176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,6144,0.03941546678543091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,5120,0.019016534090042114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,5120,0.03476053476333618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,4096,0.016114133596420287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,4096,0.0312608003616333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,512,0.007922133306662242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,3584,0.014643200238545737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1536,128,0.00613973339398702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,3584,0.02863466739654541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,3072,0.013097600142161051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,3072,0.027190399169921876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,65536,0.35160748163859046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,2560,0.01165013313293457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,2560,0.024130133787790935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,2560,0.018337066968282065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,2048,0.009876267115275065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,16384,0.09676907062530518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,2048,0.022601600488026938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,2048,0.01573013365268707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,1536,0.008285866677761078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,1536,0.020917334159215293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,1536,0.012396799524625142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,1024,0.006775466601053874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,1024,0.019358932971954346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,10240,0.060244266192118326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,768,0.006000000238418579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,768,0.01843520005544027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,512,0.005491200089454651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,512,0.017746132612228394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,8192,0.051719466845194496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,256,0.004633600016434988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,256,0.016732800006866454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,256,0.006348800162474315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,128,0.003851733356714249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,128,0.015784533818562825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,128,0.005622399846712748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,64,0.003587199995915095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,64,0.015497600038846334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,1024,32,0.003623466690381368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,1024,32,0.01602240006128947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,65536,0.14122559229532877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,5120,0.032529066006342575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,4096,0.027540266513824463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,3584,0.025093332926432295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,65536,0.333238410949707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,3072,0.021191465854644775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,51200,0.2603232065836588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,16384,0.042124799887339276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,16384,0.09147520065307617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,1024,0.010063999891281128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,12288,0.031590400139490764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,768,0.008736000458399455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,12288,0.0702303965886434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,51200,0.11529066562652587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,10240,0.02688746651013692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,1024,512,0.007567999760309856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,10240,0.05753493309020996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,8192,0.021242666244506835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,8192,0.04606506824493408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,7168,0.018578133980433145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,7168,0.040229332447052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,6144,0.016430933276812235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,6144,0.0357258677482605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,65536,0.3447786649068197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,5120,0.014218667149543762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,5120,0.03548266490300496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,12288,0.0661290685335795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,4096,0.012315733234087627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,4096,0.031193600098292036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,16384,0.09010453224182129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,51200,0.2649717330932617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,3584,0.011041067043940226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,3584,0.028627200921376543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,3072,0.010174933075904845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,3072,0.026989867289861042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,10240,0.056360534826914464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,2560,0.01286720037460327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,2560,0.024036266406377158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,7168,0.04037333329518636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,8192,0.04505279858907064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,2048,0.011122133334477742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,2048,0.02225493391354879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,6144,0.03528639872868856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,1536,0.008571733037630718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,1536,0.020037333170572914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,1024,0.006723199784755707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,1024,0.018565332889556883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,3072,0.01885333259900411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,768,0.005858133236567179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,768,0.01799573302268982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,1536,0.011161599556605022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,4096,0.02435520092646281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,5120,0.029956267277399702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,512,0.004859733581542969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,256,0.0039327998956044516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,256,0.01609280010064443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,2560,0.016260266304016113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,3584,0.02168533404668172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,128,0.0035616000493367515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,128,0.0150709331035614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,64,0.003294933338960012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,64,0.015034666657447815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,1024,0.009311999877293904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,768,32,0.003357866654793421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,32,0.015794133146603904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,65536,0.11643626689910888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,65536,0.3143978754679362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,51200,0.09877119859059652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,51200,0.2482037385304769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,768,512,0.01657386620839437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,16384,0.03190079927444458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,16384,0.09043839772542318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,2048,0.013928533593813578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,12288,0.026473599672317504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,12288,0.06791146596272787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,12288,0.06341760158538819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,10240,0.024430932601292928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,10240,0.055538133780161536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,65536,0.3062154769897461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,8192,0.021336533625920615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,8192,0.04358506600062052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,51200,0.24066559473673502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,7168,0.01798293391863505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,128,0.0055871998270352686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,7168,0.038636799653371176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,16384,0.0817845344543457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,6144,0.015895467003186545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,256,0.006215466558933258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,6144,0.03500373363494873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,5120,0.013745066523551942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,5120,0.03096746603647868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,4096,0.011934933066368104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,4096,0.026874667406082152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,512,0.0071936001380284624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,3584,0.010732799768447876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,3584,0.025497599442799883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,10240,0.05420159896214803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,3072,0.009733333190282186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,3072,0.023835732539494833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,768,768,0.008107733229796093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,2560,0.00858133335908254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,8192,0.04316480159759521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,2560,0.02188906669616699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,2560,0.01573013365268707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,2048,0.0076789334416389465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,2048,0.021478400627772013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,6144,0.033838931719462076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,7168,0.0396938681602478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,1536,0.006326400240262349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,1536,0.01940586765607198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,5120,0.02834560076395671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,1024,0.005444266895453135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,1024,0.017870932817459106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,768,0.004880000154177348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,768,0.017516799767812095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,4096,0.023653332392374674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,512,0.004308266441027323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,512,0.016609066724777223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,3584,0.021220266819000244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,256,0.003643733263015747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,256,0.01600213348865509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,3072,0.018595200777053834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,128,0.003340800106525421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,128,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,64,0.0031306666632493338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,64,0.015592533349990844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,512,32,0.003206400076548258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,512,32,0.015063466628392539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,65536,0.06725333531697592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,65536,0.29037758509318035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,65536,0.26119680404663087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,51200,0.05491840044657389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,2048,0.013270399967829385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,1536,0.010850133498509724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,51200,0.23119786580403648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,16384,0.022539732853571574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,1024,0.008920533458391826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,12288,0.015176533659299215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,51200,0.21347840627034506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,16384,0.08423999945322672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,768,0.007849599917729695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,10240,0.013230933745702108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,12288,0.055411199728647866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,8192,0.013994666934013366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,8192,0.04029866854349772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,512,0.0068800002336502075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,7168,0.015733333428700765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,7168,0.0359007994333903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,256,0.005796266595522562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,6144,0.013614933689435324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,6144,0.03285226623217265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,512,128,0.005298133194446564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,5120,0.01211840013662974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,5120,0.02966933250427246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,12288,0.06287680069605509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,4096,0.01074026624361674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,10240,0.05167359908421835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,4096,0.020171733697255452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,4096,0.026025599241256712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,3584,0.008682666222254436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,3584,0.025358933210372924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,3072,0.007944533228874206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,3072,0.023128533363342287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,16384,0.06899733543395996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,2560,0.0071274667978286745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,2560,0.021707733472188316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,2560,0.01376106639703115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,10240,0.04491093158721924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,2048,0.006512000163396199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,2048,0.011732266346613566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,2048,0.020616533358891805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,1536,0.005773866673310598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,1536,0.009891200065612792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,8192,0.038070400555928544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,1024,0.004735999802748362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,1024,0.017395200332005818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,768,0.004259199897448222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,7168,0.03163733283678691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,768,0.016832000017166136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,512,0.0037493333220481873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,512,0.015904000401496886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,512,0.006445866823196411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,256,0.0033013333876927694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,256,0.01551253298918406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,6144,0.028253867228825884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,128,0.0031285333136717476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,128,0.015238400300343832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,5120,0.02368639906247457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,64,0.0029418667157491045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,64,0.015261866648991904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,256,32,0.0029728000362714132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,32,0.01497706671555837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,3584,0.018361600240071614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,256,1536,0.01881493330001831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,65536,0.058650668462117514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,3072,0.015793066223462424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,51200,0.04728000164031983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,65536,0.28352320988972984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,51200,0.16154239972432455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,16384,0.01515733301639557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,51200,0.22471680641174316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,16384,0.055118934313456214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,16384,0.0801589330037435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,12288,0.016386133432388306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,12288,0.05790613492329916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,12288,0.04240959882736206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,10240,0.014230400323867798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,10240,0.04839680194854736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,10240,0.03561066786448161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,8192,0.012101333340009053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,8192,0.03826560179392497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,8192,0.028867199023564653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,7168,0.010479999581972758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,7168,0.033432531356811526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,6144,0.010168533523877461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,6144,0.03109546701113383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,6144,0.022743467489878336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,5120,0.00997759997844696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,5120,0.02806719938913981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,1024,0.00804906686147054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,4096,0.008588799834251403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,768,0.007353599866231282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,4096,0.02617066701253255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,3584,0.007844266792138416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,3584,0.023763199647267662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,256,0.00557226687669754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,3072,0.007169066866238911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,3072,0.02230186661084493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,256,128,0.005137066543102265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,3072,0.013733333349227906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,2560,0.006484266618887584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,2560,0.020705066124598184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,2048,0.005635199944178263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,2048,0.019347200791041054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,2048,0.010657067100207012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,1536,0.004914133250713349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,1536,0.018154666821161906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,1536,0.009526399771372478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,1024,0.004271999994913737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,1024,0.01773653427759806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,1024,0.007842133442560833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,65536,0.2061631997426351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,768,0.0038453333079814913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,768,0.016919465859731038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,512,0.0035861333211263022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,512,0.006832000116507213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,512,0.016377600034077962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,256,0.003271466741959254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,256,0.006126933296521505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,256,0.01525759994983673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,128,0.0029877332349618276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,128,0.015525333086649575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,128,0.00551146666208903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,64,0.002914133419593175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,64,0.014842666188875833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,128,32,0.002997333308060964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,128,32,0.014945066968599954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,65536,0.05554560025533041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,51200,0.04424959818522135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,65536,0.2817930539449056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,16384,0.013662933309872945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,51200,0.22258240381876626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,5120,0.0198527991771698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,12288,0.01162559986114502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,16384,0.07798399925231933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,12288,0.054679465293884275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,8192,0.008524800340334576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,10240,0.044232531388600664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,7168,0.00826453318198522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,8192,0.03575679858525594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,6144,0.00784853349129359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,7168,0.03221866687138875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,6144,0.02998720010121663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,4096,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,5120,0.007158400118350982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,4096,0.0072202667593956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,5120,0.02770986755688985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,4096,0.02494186758995056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,3584,0.006814933319886525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,3072,0.0064085334539413456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,3584,0.023834667603174844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,3072,0.022616533438364665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,3584,0.01595200002193451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,2048,0.005497600138187409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,2560,0.020966400702794395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,1536,0.004834133386611939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,2048,0.019717333714167277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,1024,0.004194133480389913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,1536,0.018360533316930137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,768,0.0038399999340375268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,1024,0.016774400075276693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,768,0.016272000471750894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,256,0.003065599997838338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,512,0.016103466351826988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,128,0.002889599899450938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,256,0.01530239979426066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,128,0.014917332927385965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,64,0.002738133321205775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,32,0.0027882667879263563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,64,0.01508799990018209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,64,32,0.014651733636856078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,65536,0.058531200885772704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,2560,0.012569600343704223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,51200,0.04581120014190674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,10240,0.010145066181818645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,65536,0.28067839940389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,16384,0.012640000383059184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,12288,0.011885866522789001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,51200,0.22224747339884438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,16384,0.07607893149058023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,12288,0.0543936014175415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,10240,0.010110933581988018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,8192,0.008889599641164144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,10240,0.04296319882074992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,7168,0.008321066697438557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,8192,0.03477866649627685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,6144,0.0073728000124295544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,5120,0.006779733300209046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,7168,0.03214079936345418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,6144,0.02995520035425822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,4096,0.00622506688038508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,5120,0.027821866671244304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,2560,0.0061589335401852924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,4096,0.02528746724128723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,3584,0.024295467138290405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,3072,0.023511467377344768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,2560,0.020821332931518555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,64,512,0.0034175999462604523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,2048,0.005515733361244201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,7168,0.025923200448354083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,1536,0.004780800143877665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,2048,0.020172800620396933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,1536,0.018212266763051353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,1024,0.01714026729265849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,768,0.0037632000943024955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,512,0.0034346667428811393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,768,0.016532267133394875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,256,0.003160533308982849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,512,0.016359466314315795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,128,0.002845866729815801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,256,0.015452800194422403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,64,0.0027274665733178455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,128,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,32,0.002726399898529053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,64,0.014907733599344889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1025,32,32,0.014794666568438211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,3584,0.006977066894372304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,3072,0.006566399832566579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,2560,0.006123733520507812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1025,32,1024,0.00407679999868075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1025,128,768,0.0074080000321070354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,16384,1.6099658966064454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,16384,1.9238005320231122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,12288,1.1832416534423829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,12288,2.231059265136719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,16384,3.2517601013183595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,12288,1.5542346954345703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,10240,1.079471969604492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,10240,1.913600031534831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,10240,1.2369098663330078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,8192,0.851901880900065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,8192,1.4453333536783854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,51200,5.169041951497396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,8192,0.9589877446492514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,7168,0.6974112192789714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,7168,0.8783999760945639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,7168,1.408344523111979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,6144,1.2018901824951171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,6144,0.6333887736002605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,6144,0.7512736002604167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,51200,5.961711120605469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,5120,0.5535519917805989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,5120,1.0003050486246745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,4096,0.42926613489786786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,5120,0.5866890589396159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,4096,0.5338282903035482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,4096,0.771504020690918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,3584,0.40017067591349287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,3584,0.4785951932271321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,3072,0.5755754470825195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,3584,0.7039765040079753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,3072,0.3951157251993815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,2560,0.2922730763753255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,2560,0.5007957458496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,2560,0.334224001566569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,2048,0.2346794605255127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,2048,0.4349482536315918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,1536,0.19836799303690594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,1536,0.3077109336853027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,1536,0.242522668838501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,1024,0.20062294006347656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,1024,0.1568554719289144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,768,0.1715775966644287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,1024,0.18328426678975424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,768,0.1217952013015747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,512,0.13027093410491944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,768,0.1636799971262614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,256,0.0815605322519938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,512,0.13051946957906085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,256,0.0788586695988973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,256,0.10969493389129639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,128,0.1078378677368164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,128,0.07293439706166585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,64,0.05866133371988932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,64,0.07320106824239095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,32,0.05848533312479655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,3072,0.3489205360412598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,32,0.0722879966100057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,51200,9.575051879882812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,512,0.09864853223164877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,128,0.06347306569417319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,2048,0.26471039454142253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,16384,1.2555989583333333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,16384,2.4482762654622396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,51200,4.116275278727214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,65536,5.136690266927084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,16384,1.4769493103027345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,51200,4.546497090657552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,12288,1.7538548787434896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,12288,0.9553909301757812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,12288,1.0667007446289063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,65536,6.857083638509114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,10240,0.77598082224528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,10240,0.8956725438435873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,10240,1.5096970876057942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,8192,0.6380053202311198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,8192,1.2668042500813803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,8192,0.7329834620157878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,7168,0.5435349146525066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,7168,1.0264426549275716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,7168,0.6468245188395183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,51200,7.68624979654948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,6144,0.5236650784810384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,5120,0.4208970705668132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,6144,0.548848025004069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,5120,0.7166090647379557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,6144,0.9142261505126953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,5120,0.4548010508219401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,4096,0.3521802584330241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,4096,0.5782538731892903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,4096,0.39220266342163085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,3584,0.3117685317993164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,3584,0.3664736111958822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,3584,0.5470762888590495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,3072,0.2702687899271647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,3072,0.44750932057698567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,3072,0.29397013982137044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,2560,0.3760074615478516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,2560,0.22796799341837565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,65536,9.840682983398438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,2048,0.19189546902974447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,2048,0.3263317426045736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,2560,0.26110186576843264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,2048,0.24300586382548012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,1536,0.1600074609120687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,1536,0.22505599657694497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,1536,0.178874667485555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,768,0.12458240191141765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,1024,0.15165759722391764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,1024,0.11550613244374593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,768,0.10070186456044514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,512,0.09582826296488443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,768,0.13007786273956298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,512,0.07972479661305745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,1024,0.16552640597025553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,256,0.0661621332168579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,512,0.10317333539326985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,128,0.05187733173370361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,128,0.06044053236643473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,64,0.053707734743754065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,51200,32,0.04743146498998006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,256,0.09321599801381429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,256,0.06596479813257852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,51200,128,0.07993280092875163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,64,0.06671466827392578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,51200,32,0.061102934678395594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,16384,0.4180266698201497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,16384,0.6870527903238932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,16384,0.459112548828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,12288,0.559764289855957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,51200,1.3302837371826173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,12288,0.3439626693725586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,51200,1.495239512125651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,12288,0.36625493367513023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,65536,1.6900746663411457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,10240,0.2762218793233236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,10240,0.48533334732055666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,65536,2.0256234486897786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,8192,0.36776854197184244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,10240,0.2988373438517252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,51200,2.4200714111328123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,8192,0.23725226720174156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,8192,0.23313706715901694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,7168,0.32132587432861326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,7168,0.21572480201721192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,6144,0.2678175926208496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,7168,0.23719573020935059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,6144,0.1816149393717448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,6144,0.18426772753397624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,5120,0.22312960624694825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,5120,0.15849599838256836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,4096,0.13533226648966473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,3584,0.1560490608215332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,3584,0.11436800161997478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,65536,3.2996884663899744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,3584,0.11497920354207355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,3072,0.09901653130849203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,3072,0.13515839576721192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,2560,0.11616960366566975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,2560,0.08512639999389648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,3072,0.1189184029897054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,2048,0.0949951966603597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,2048,0.07296960353851319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,1536,0.07373119990030924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,2048,0.08161919911702474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,1536,0.06042240063349406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,1024,0.05656319856643677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,1536,0.06322773297627768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,1024,0.04967999855677287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,768,0.04282559951146443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,1024,0.05443413257598877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,768,0.04384640057881673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,512,0.03542933464050293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,768,0.04408746560414632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,512,0.03746453523635864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,256,0.023155200481414794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,512,0.041323733329772946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,128,0.019179733594258626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,256,0.03168746630350749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,128,0.031146667400995892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,64,0.015468800067901611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,4096,0.18695680300394696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,32,0.015508266290028891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,64,0.029554132620493574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,32,0.029971200227737426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,2560,0.08513173262278238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,256,0.03214933276176453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,4096,0.12632426420847576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,65536,1.300378672281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,65536,1.464462916056315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,65536,2.2379689534505207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,16384,0.37742080688476565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,51200,1.0051040013631185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,5120,0.1553610642751058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,16384,0.3541066805521647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,51200,1.1128565470377603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,51200,1.7088757832845052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,12288,0.4040202776590983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,12288,0.2712405204772949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,12288,0.26276373863220215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,10240,0.33808533350626624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,8192,0.17864960034688313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,10240,0.23891733487447103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,8192,0.27024106979370116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,8192,0.18442773818969727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,10240,0.24434773127237955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,7168,0.1565685272216797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,7168,0.16954666773478191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,6144,0.20621652603149415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,6144,0.1433344046274821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,6144,0.1482026735941569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,5120,0.16575679779052735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,5120,0.1271989345550537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,5120,0.12101759910583496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,4096,0.0964202642440796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,3584,0.12128853003184001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,3584,0.09343146483103434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,3584,0.0855242649714152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,3072,0.10545813242594401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,3072,0.0807039976119995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,3072,0.07543253103892009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,2560,0.09005226294199625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,2560,0.07121919790903727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,2560,0.06576746702194214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,2048,0.07464106877644858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,2048,0.06063893238703409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,1536,0.058202668031056726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,2048,0.05740480025609335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,1536,0.052738134066263834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,1024,0.040185598532358806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,1024,0.04272426764170329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,768,0.03389120101928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,768,0.03777173360188802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,768,0.034430932998657224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,128,0.02885546684265137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,512,0.025810132424036663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,7168,0.24746346473693848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,512,0.029080533981323244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,512,0.03284586668014526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,256,0.01818880041440328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,128,0.014235732952753702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,256,0.024609067042668662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,16384,0.5314325332641602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,128,0.022129066785176597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,128,0.02661973237991333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,4096,0.13603413899739583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,4096,0.10328959623972575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,32,0.012974933783213297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,64,0.012637866536776224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,64,0.026203733682632447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,32,0.025659734010696413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,1536,0.051215998331705724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,256,0.02840106685956319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,65536,1.1381749471028646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,65536,1.1774283091227213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,51200,1.4221983591715495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,51200,0.906008529663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,16384,0.45720427831013993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,16384,0.32864532470703123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,65536,1.9807348887125653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,16384,0.30837653477986654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,12288,0.3381578763326009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,12288,0.2379701296488444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,12288,0.23527785936991372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,10240,0.19374613761901854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,51200,0.8802464167277018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,10240,0.30282347997029624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,10240,0.20141332944234214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,8192,0.21896746953328453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,8192,0.1566175937652588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,8192,0.15547839800516766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,7168,0.2220597267150879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,7168,0.15285332997639972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,7168,0.14144213994344074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,6144,0.1665621280670166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,6144,0.120087464650472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,5120,0.1606837272644043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,5120,0.10685973167419434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,4096,0.11408212979634602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,4096,0.09138773282368978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,3584,0.10197439988454182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,3584,0.08108800252278646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,3072,0.08749866485595703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,3584,0.08227519989013672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,3072,0.0727189302444458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,2560,0.07876693407694499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,3072,0.0678816000620524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,2560,0.062344535191853845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,2560,0.06392960151036581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,2048,0.05448640187581381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,2048,0.0507584015528361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,1536,0.04732480049133301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,1536,0.0420415997505188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,1024,0.0348906675974528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,1024,0.036297599474589035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,1024,0.0333024005095164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,768,0.027797333399454754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,768,0.03245439926783244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,768,0.02991360028584798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,512,0.020770132541656494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,512,0.02839786609013875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,512,0.025050665934880572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,5120,0.1044917345046997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,256,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,128,0.011282133062680562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,256,0.02140373388926188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,128,0.02167466680208842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,128,0.01963520050048828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,64,0.009377066294352214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,64,0.022006400426228843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,32,0.009518933296203614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,32,0.022451200087865195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,2048,0.060743467013041175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,1536,0.0458346684773763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,4096,0.08503466447194417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,256,0.024600533644358318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,65536,0.9634677251180014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,65536,1.009722646077474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,65536,1.5124170939127604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,51200,1.0444149017333983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,16384,0.35226879119873045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,16384,0.29194558461507164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,12288,0.2965653419494629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,16384,0.26434453328450525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,12288,0.19960853258768718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,51200,0.7311871846516926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,12288,0.21683306694030763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,51200,0.752403195699056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,10240,0.1667082627614339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,10240,0.16904533704121907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,8192,0.1319808006286621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,8192,0.17661546071370443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,8192,0.1478805383046468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,7168,0.15685973167419434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,7168,0.117412265141805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,7168,0.11508906682332357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,6144,0.13457279205322265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,5120,0.1107413371404012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,6144,0.11764053503672282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,5120,0.09053653081258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,5120,0.08694079717000326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,4096,0.08879146575927735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,4096,0.08726720015207926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,4096,0.07187626361846924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,3584,0.06766080061594645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,3072,0.06931946277618409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,3584,0.07247146765391031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,3072,0.05959999958674113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,3072,0.0552512009938558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,2560,0.053185065587361656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,2560,0.04729493459065755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,2048,0.04758826494216919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,2048,0.04520639975865682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,2048,0.041510399182637533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,1536,0.03727680047353109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,1536,0.03778133392333984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,1536,0.03421866496404012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,10240,0.23843626976013182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,1024,0.0264682670434316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,1024,0.027764266729354857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,1024,0.0323360006014506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,768,0.021577600638071695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,768,0.028908799091974895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,768,0.024870399634043375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,512,0.016898133357365928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,512,0.02558506727218628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,6144,0.10356693267822266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,256,0.011971199512481689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,256,0.01781866749127706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,256,0.021596799294153847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,128,0.008853333195050557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,3584,0.08154880205790202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,128,0.020295466979344687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,64,0.007580799857775371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,128,0.016475733121236166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,64,0.020515199502309164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,32,0.020718934138615926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,2560,0.05845119953155518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,1024,0.038099201520284016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,65536,0.9176661173502604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,65536,0.9800095876057944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,51200,1.003870964050293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,32,0.008198399841785432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,65536,1.427078374226888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,16384,0.23531947135925294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,51200,0.7302666982014974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,12288,0.2435274600982666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,12288,0.1875498612721761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,10240,0.20501012802124025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,10240,0.1529813289642334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,10240,0.15772159894307453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,8192,0.16650667190551757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,8192,0.12456320126851399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,7168,0.14478826522827148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,16384,0.36432533264160155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,6144,0.11775360107421876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,7168,0.11109120051066082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,6144,0.1264906644821167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,7168,0.11233173211415608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,6144,0.09776960213979086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,5120,0.105732266108195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,5120,0.08505386511484782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,4096,0.08493759632110595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,4096,0.07124799887339274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,16384,0.25011733373006184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,3584,0.07658452987670898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,51200,0.784060796101888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,3584,0.06573013464609781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,3072,0.06650453408559164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,12288,0.18325333595275878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,3072,0.058362666765848795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,2560,0.056391465663909915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,2560,0.052179201443990075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,512,0.021223467588424683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,2048,0.045425065358479816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,8192,0.12566080093383789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,2048,0.0434282660484314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,1536,0.03566399812698364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,1536,0.03662826617558797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,6144,0.09528960386912028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,1024,0.02568746606508891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,5120,0.08105706373850505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,1024,0.031396265824635824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,768,0.02101973295211792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,1024,0.025540266434351606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,768,0.027961599826812743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,512,0.016379732886950174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,512,0.024572799603144325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,4096,0.06679360071818033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,256,0.01169706682364146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,256,0.02206719915072123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,256,0.013977600137392679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,128,0.008564266562461852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,128,0.0206496000289917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,3584,0.059106131394704185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,64,0.007368533313274384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,3072,0.05125120083491007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,32,0.008160000046094257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,64,0.021222400665283202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,32,0.02113920052846273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,2560,0.04632746775945028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,2048,0.03783146540323894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,1536,0.030115199089050294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,65536,0.7850122451782227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,65536,0.780505625406901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,65536,1.1448095957438151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,768,0.020538665850957236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,16384,0.3113674799601237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,512,0.01725333333015442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,51200,0.8285247802734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,16384,0.1837706724802653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,16384,0.22862292925516764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,12288,0.23495039939880372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,12288,0.15288426081339518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,10240,0.16682772636413573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,51200,0.577791976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,128,0.012402133146921793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,10240,0.11995840072631836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,10240,0.1512074629465739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,8192,0.1342591921488444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,8192,0.09629546801249186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,8192,0.11463680267333984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,7168,0.11881279945373535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,7168,0.09021013577779134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,6144,0.10111146767934162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,6144,0.0755125363667806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,6144,0.09102293650309244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,5120,0.0859392007191976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,5120,0.07308266957600912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,4096,0.06928426424662272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,4096,0.06435946623484293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,4096,0.053140266736348474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,3584,0.06128533283869425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,3584,0.058100267251332605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,3072,0.053041064739227296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,3584,0.047722665468851726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,3072,0.05235199928283692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,3072,0.04374826749165853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,2560,0.04576319853464762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,2560,0.03847039937973022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,2048,0.03712533315022786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,51200,0.6132330576578776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,2048,0.03938453197479248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,2048,0.033532798290252686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,1536,0.02813120086987813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,1024,0.020282665888468422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,1536,0.027765333652496338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,1536,0.03309440016746521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,1024,0.022641066710154215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,768,0.017256534099578856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,1024,0.027480532725652058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,12288,0.15665173530578613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,512,0.013303466637929282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,768,0.020433066288630168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,256,0.009201066692670186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,512,0.017181867361068727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,512,0.023170133431752525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,128,0.007353599866231282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,256,0.014293332894643148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,256,0.02004800041516622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,7168,0.10229439735412597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,64,0.006412800153096516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,128,0.019062399864196777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,128,0.013352533181508383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,32,0.0066431999206542965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,64,0.019453867276509603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,32,0.01922453244527181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,5120,0.06360426743825277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,2560,0.04486720164616902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,51200,0.5367488225301107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,51200,0.6641216278076172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,65536,0.6804842631022135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,65536,0.8322378794352213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,768,0.025512532393137617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,12288,0.17947840690612793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,16384,0.19922560056050617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,16384,0.19918932914733886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,51200,0.6229546864827473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,10240,0.13671253522237142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,12288,0.13493226369222006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,8192,0.11958506902058919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,8192,0.09833813508351644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,7168,0.10595413049062093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,7168,0.08877226511637369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,6144,0.08377172946929931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,6144,0.07700160344441732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,16384,0.21473280588785806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,5120,0.07008746465047201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,5120,0.06536853313446045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,10240,0.11503252983093262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,4096,0.056260267893473305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,4096,0.05558826526006063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,65536,0.7871327718098958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,3584,0.05680319865544638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,3584,0.05475946664810181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,12288,0.14571305910746257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,3072,0.04302613337834676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,10240,0.13086613019307453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,8192,0.09913492997487386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,3072,0.045672531922658285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,3072,0.03995840152104695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,2560,0.037170132001241044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,2560,0.040326400597890215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,2048,0.030825599034627275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,2048,0.035086933771769205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,1536,0.024195200204849242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,1536,0.030933332443237305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,1536,0.02430079976717631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,1024,0.0180565337340037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,6144,0.07415359814961751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,1024,0.026694399118423463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,5120,0.06396906773249308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,768,0.015229866902033488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,768,0.024997333685557045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,7168,0.08614292939503988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,512,0.012155733505884806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,4096,0.05201493501663208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,512,0.022060799598693847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,512,0.013727999726931252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,256,0.008614400029182434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,256,0.01143999993801117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,256,0.019738666216532388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,128,0.007227733234564463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,128,0.01846826672554016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,64,0.006276266773541768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,64,0.018833067019780478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,32,0.006478933493296306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,32,0.01867093245188395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,65536,0.6634613037109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,3584,0.047313066323598225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,2560,0.036339199542999266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,2048,0.030237867434819536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,65536,0.5388128280639648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,65536,0.6040384292602539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,1024,0.02060479919115702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,51200,0.5193365414937336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,16384,0.16883625984191894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,51200,0.46283626556396484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,16384,0.1560245354970296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,12288,0.12792426745096844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,16384,0.1426794687906901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,12288,0.11552213033040364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,51200,0.4397909482320149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,12288,0.10317760308583576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,10240,0.10959786574045818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,10240,0.08673706849416098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,10240,0.10081173578898113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,8192,0.07520853678385417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,8192,0.09364480177561442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,7168,0.07553066412607828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,6144,0.07188693682352701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,7168,0.06680320103963217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,6144,0.06730986436208089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,5120,0.05797653198242188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,5120,0.04657599925994873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,5120,0.05825066566467285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,4096,0.046187734603881835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,768,0.01684479912122091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,3584,0.040085331598917646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,4096,0.04833600123723348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,3072,0.034306132793426515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,3584,0.036686933040618895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,3584,0.043426132202148436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,2560,0.029894399642944335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,3072,0.03287786642710368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,2560,0.02881386677424113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,2560,0.035011200110117595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,2048,0.024842667579650878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,128,0.009966933727264404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,2048,0.03226986726125081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,2048,0.025016532341639204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,1536,0.027268266677856444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,1536,0.021242666244506835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,1024,0.015102932850519816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,1024,0.02441706657409668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,1024,0.017051732540130614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,768,0.012611200412114462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,768,0.015656532843907674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,768,0.02188906669616699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,512,0.010114133358001709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,512,0.019785600900650024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,512,0.013491200407346091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,256,0.007344000041484833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,256,0.018339200814565023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,256,0.011422933141390482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,128,0.006151466568311056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,8192,0.08230613072713217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,7168,0.0769162654876709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,128,0.01769066651662191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,128,0.010620799660682679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,32,0.005622399846712748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,64,0.01795733372370402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,32,0.017540266116460167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,3072,0.039293865362803146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,65536,0.5725226720174154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,65536,0.6367605209350586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,1536,0.01984106699625651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,16384,0.15851200421651204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,51200,0.48136212031046555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,16384,0.15211413701375326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,51200,0.45218880971272785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,12288,0.12021013100941975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,64,0.0054282665252685545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,12288,0.11391466458638508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,10240,0.10109546979268391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,10240,0.0958304007848104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,8192,0.08219412962595621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,8192,0.08063680330912272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,8192,0.07095466454823812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,7168,0.07241386572519938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,7168,0.07314240137736003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,65536,0.5354111989339192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,7168,0.07073173522949219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,6144,0.06073386669158935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,6144,0.07076906363169352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,16384,0.13812905947367352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,51200,0.42277971903483075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,5120,0.05223893324534098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,12288,0.10355733235677082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,5120,0.06149760087331137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,5120,0.04684266646703084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,4096,0.039943468570709226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,4096,0.03955093224843343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,3584,0.03606719970703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,4096,0.0482805331548055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,3584,0.04060693184534709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,3072,0.03095146616299947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,3072,0.03681066830952962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,2560,0.025671466191609697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,10240,0.08936746915181479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,2560,0.032818132638931276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,2048,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,2048,0.029474133253097536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,1536,0.017589332660039265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,1536,0.02542613347371419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,3072,0.032613333066304526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,1024,0.012537599603335062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,1024,0.02111466725667318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,6144,0.05460160175959269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,4096,0.03930559953053792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,768,0.010429867108662923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,768,0.020055466890335084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,512,0.007969066500663757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,512,0.0190175990263621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,3584,0.03475840091705322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,256,0.006308266520500183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,6144,0.055707732836405434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,256,0.018202666441599527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,128,0.005350400010744731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,128,0.01722453236579895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,2560,0.027435733874638872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,64,0.004951466619968414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,64,0.01786880095799764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,32,0.005275733272234599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,32,0.017794134219487508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,1536,0.019181867440541588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,65536,0.5240074793497722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,1024,0.015280000368754067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,65536,0.5141493479410808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,2048,0.02304319938023885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,51200,0.40743147532145185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,51200,0.407256539662679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,16384,0.13680532773335774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,16384,0.13596800168355305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,12288,0.10004693667093914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,12288,0.10168426831563313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,768,0.013401599725087485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,10240,0.08314879735310873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,512,0.011426132917404175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,10240,0.0862666686375936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,8192,0.06704959869384766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,256,0.00957546631495158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,8192,0.07266133626302083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,128,0.008629332979520161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,7168,0.05911360184351603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,7168,0.0660042683283488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,6144,0.05076906681060791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,6144,0.05873706738154093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,65536,0.47456960678100585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,5120,0.04371946652730306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,5120,0.05197013219197592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,5120,0.044494934876759845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,4096,0.035665067036946614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,4096,0.04311360120773315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,51200,0.3741728146870931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,3584,0.03081600069999695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,3584,0.038533333937327066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,3584,0.03227519989013672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,16384,0.12256533304850262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,3072,0.026842667659123735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,3072,0.03484586477279663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,2560,0.023361066977183022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,2560,0.03158933321634928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,12288,0.09287253220876059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,2048,0.01941439906756083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,2048,0.028282666206359865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,10240,0.07781866391499838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,1536,0.015867732961972556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,8192,0.0657482663790385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,1536,0.025461333990097045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,1024,0.012050132950146992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,1024,0.020796799659729005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,1024,0.014454399545987448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,7168,0.055870934327443444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,768,0.01018986701965332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,768,0.020550400018692017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,6144,0.048446933428446456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,512,0.007950933277606964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,512,0.019308799505233766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,256,0.006180266539255777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,256,0.01779200037320455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,4096,0.03601386547088623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,256,0.009612799684206644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,128,0.0053045332431793215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,128,0.01708266735076904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,3072,0.027914667129516603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,64,0.004725333551565806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,64,0.01697280009587606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,32,0.004845866560935974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,2560,0.025131734212239583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,32,0.017077332735061644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,2048,0.02151040037473043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,1536,0.01795733372370402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,65536,0.43972266515096026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,65536,0.482755184173584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,768,0.012785067160924276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,51200,0.3515647888183594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,512,0.011534933249155681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,16384,0.11145066420237223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,51200,0.389030392964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,12288,0.08505813280741373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,16384,0.12870399951934813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,12288,0.09699412981669107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,10240,0.07315200169881185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,8192,0.05733866691589355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,10240,0.08328533172607422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,8192,0.06846186319986979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,7168,0.05071573257446289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,7168,0.06179840167363485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,128,0.008728532989819845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,6144,0.04459306796391805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,6144,0.056347731749216715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,12288,0.0853664000829061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,5120,0.03768320083618164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,5120,0.04699093500773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,65536,0.4386912027994792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,4096,0.030943999687830608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,4096,0.04059520165125529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,16384,0.11413546403249104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,3584,0.02736213405927022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,10240,0.07412160237630208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,3584,0.0361685315767924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,3072,0.02386346658070882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,3072,0.033124266068140666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,7168,0.052425599098205565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,2560,0.020853332678476968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,2560,0.030462932586669923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,51200,0.3460618654886881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,6144,0.045544532934824626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,2048,0.017927465836207072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,2048,0.027665066719055175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,1536,0.014699733257293702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,8192,0.059214933713277185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,1536,0.024589866399765015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,1024,0.011458133657773335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,1024,0.021489065885543824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,3072,0.02577706575393677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,768,0.009361066420873006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,768,0.020520534118016562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,4096,0.03279786705970764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,512,0.007794133325417836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,512,0.019290665785471596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,5120,0.03993813196818034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,256,0.005975466469923655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,256,0.017518933614095053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,1536,0.016035200158754984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,128,0.005050666630268097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,128,0.01687999963760376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,3584,0.029890133937199907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,64,0.00458133320013682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,64,0.01684266726175944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,32,0.0048320000370343525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,1024,0.013386666774749756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,32,0.016874667008717856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,65536,0.34588588078816734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,65536,0.4284554799397786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,2048,0.01990613341331482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,65536,0.3323424021402995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,51200,0.27643092473347985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,16384,0.09018026987711589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,51200,0.3334613482157389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,51200,0.26122026443481444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,2560,0.023423999547958374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,12288,0.06722239653269449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,16384,0.08261546293894449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,12288,0.08541759649912516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,10240,0.05515306790669759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,12288,0.06649706761042276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,10240,0.0738090674082438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,8192,0.04588479995727539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,10240,0.053651201725006106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,7168,0.03958400090535481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,8192,0.061826133728027345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,8192,0.04745920101801555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,6144,0.03514986832936605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,7168,0.03974293470382691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,7168,0.055523200829823816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,5120,0.029063467184702558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,6144,0.05028906663258871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,5120,0.031058132648468018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,5120,0.04149973392486572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,4096,0.02412266731262207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,4096,0.035973334312438966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,16384,0.1148799975713094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,3584,0.021486934026082358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,3584,0.032605866591135665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,128,0.007889066636562348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,3072,0.01871253252029419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,3072,0.029894399642944335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,256,0.008781866232554118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,512,0.010297600428263347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,2560,0.02717439929644267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,2560,0.018359466393788656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,2048,0.013821867108345032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,2048,0.024653865893681844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,768,0.011574400464693706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,1536,0.011311999956766764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,1536,0.02198186715443929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,1536,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,1024,0.008538666367530822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,1024,0.02039253314336141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,1024,0.011361066500345867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,768,0.0074325333038965866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,768,0.01882986625035604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,768,0.010654933253924052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,512,0.0061930666367212926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,512,0.01766293247540792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,512,0.009666132926940917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,256,0.0052149335543314615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,2560,0.016328533490498863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,256,0.008288000027338665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,256,0.016708266735076905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,128,0.00451200008392334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,128,0.01630826691786448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,128,0.007609599828720092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,32,0.004129066566626231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,64,0.016369066635767617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,32,0.016127999623616537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,65536,0.2641770680745443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,65536,0.3915893236796061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,51200,0.2104490598042806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,51200,0.3123648007710775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,64,0.004175999760627746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,16384,0.06742613315582276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,51200,0.23430293401082358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,16384,0.10307306448618572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,16384,0.07671146392822266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,12288,0.05123733282089234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,12288,0.08184746901194254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,12288,0.05789546569188436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,10240,0.043136000633239746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,10240,0.06771519978841146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,8192,0.03511893351872762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,8192,0.05503360033035278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,8192,0.041317331790924075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,7168,0.03092693289120992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,7168,0.04844906727472941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,65536,0.2931040128072103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,6144,0.027154133717219038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,6144,0.04283200105031331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,4096,0.025522132714589436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,5120,0.0228383998076121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,5120,0.03727893431981404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,6144,0.034663466612497966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,4096,0.01845653255780538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,4096,0.03197120030721028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,10240,0.05027519861857096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,3584,0.01716053287188212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,3584,0.029683200518290202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,3584,0.0218560000260671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,3072,0.015802666544914246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,3072,0.028037333488464357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,7168,0.036525865395863846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,2560,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,2560,0.02411839962005615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,2560,0.016926934321721397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,2048,0.011193600296974183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,2048,0.021948800484339396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,3584,0.02334933280944824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,1536,0.00921066701412201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,1536,0.02074133356412252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,1536,0.012098133563995361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,1024,0.007216000060240428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,1024,0.018962132930755615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,1024,0.010154666503270467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,768,0.006490666667620342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,768,0.01835413376490275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,2048,0.01637333333492279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,512,0.00544106662273407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,512,0.01742186745007833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,6144,0.03181440035502116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,512,0.008598400155703227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,256,0.016567466656366985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,256,0.007277866701285045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,128,0.004018133382002512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,128,0.01588159998257955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,128,0.0067071999112765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,64,0.003735466549793879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,64,0.01591146687666575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,32,0.003851733356714249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,32,0.016064000129699708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,3072,0.020933334032694498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,65536,0.17946453094482423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,5120,0.028538666168848675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,4096,0.023283199469248454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,65536,0.3461578687032064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,51200,0.14164586067199708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,256,0.0045973335703214015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,16384,0.05057599941889444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,3072,0.019192532698313395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,51200,0.27365973790486653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,16384,0.09810986518859863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,12288,0.04553066492080689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,12288,0.0785045305887858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,10240,0.040550398826599124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,10240,0.06503786643346152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,2048,0.014326399564743042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,8192,0.027398399511973065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,8192,0.050221868356068934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,768,0.009278933207194012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,7168,0.02889813383420308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,7168,0.044420266151428224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,6144,0.0256330668926239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,6144,0.040387201309204104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,51200,0.20018879572550455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,65536,0.25192532539367674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,5120,0.035426131884257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,16384,0.0683018684387207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,4096,0.016747732957204185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,4096,0.0303274671236674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,12288,0.051726933320363364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,3584,0.014682666460673014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,3584,0.02789546648661296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,10240,0.04545386632283528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,3072,0.014301866292953491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,3072,0.025385600328445435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,8192,0.036483200391133626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,2560,0.01234879990418752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,2560,0.02404159903526306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,5120,0.022819199164708457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,7168,0.0322655995686849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,2048,0.010172800223032633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,2048,0.02125119964281718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,1536,0.007628799974918365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,1536,0.01983039975166321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,6144,0.028486400842666626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,1024,0.006150400141874949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,1024,0.018550399939219156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,4096,0.02132373253504435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,5120,0.024754132827123007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,768,0.005636266867319743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,768,0.0178656001885732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,512,0.004785066843032837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,512,0.01667733391125997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,3072,0.016807466745376587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,256,0.004145066688458125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,256,0.01625599960486094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,2048,0.012812800208727517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,128,0.003702399879693985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,128,0.015172266960144043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,3584,0.019141334295272826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,64,0.0034143999218940735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,64,0.015461333592732749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,32,0.003909333298603693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,32,0.015448533495267234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,65536,0.13994240760803223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,1024,0.009313066800435383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,65536,0.3258410771687826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,2560,0.015052800377209982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,51200,0.11233920256296795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,51200,0.25949974060058595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,16384,0.04054506619771321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,1536,0.01130560040473938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,16384,0.09644906520843506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,12288,0.04107840061187744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,12288,0.07386879920959473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,256,0.00697386662165324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,10240,0.036321067810058595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,10240,0.060838401317596436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,512,0.008110933502515157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,8192,0.02577173312505086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,768,0.008836266398429871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,128,0.0066336000959078475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,8192,0.04774080117543538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,7168,0.022032000621159873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,7168,0.042131201426188154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,6144,0.020959999163945517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,6144,0.03750186761220296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,65536,0.24010666211446127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,12288,0.04829119841257731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,5120,0.018578133980433145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,5120,0.033214932680130003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,4096,0.014713600277900696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,4096,0.028967465957005816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,16384,0.0631114681561788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,3584,0.012572800119717916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,3584,0.026522666215896606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,51200,0.18935359319051107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,3072,0.011338667074839274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,3072,0.02468693256378174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,10240,0.041739734013875325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,6144,0.026077866554260254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,2560,0.009892266988754273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,2560,0.022820266087849934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,7168,0.029857067267100017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,2048,0.008362666765848795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,2048,0.020894932746887206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,8192,0.03453546762466431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,1536,0.006934399902820587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,1536,0.01984000007311503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,1024,0.0056618665655454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,1024,0.018649599949518838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,3072,0.015201066931088766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,768,0.004780800143877665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,768,0.016992000738779704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,4096,0.01945706605911255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,512,0.004281599819660187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,512,0.016671999295552572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,3584,0.017203199863433837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,256,0.0037269333998362223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,256,0.015553067127863566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,5120,0.022737065951029457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,128,0.0034304000437259674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,128,0.015238400300343832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,1536,0.010293333729108175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,2560,0.013951999942461648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,64,0.003201066702604294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,32,0.0033088001112143196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,64,0.015093333522478738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,32,0.015428266922632852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,65536,0.09883519808451334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,65536,0.3047957420349121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,51200,0.07751466433207194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,2048,0.011895466844240825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,51200,0.2423744042714437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,16384,0.03296213348706563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,16384,0.09084160327911377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,512,0.007150933146476746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,12288,0.034058666229248045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,256,0.006298666695753734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,768,0.0077802668015162155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,10240,0.030050132671991987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,10240,0.05687893231709799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,1024,0.008549333612124125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,8192,0.02099626660346985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,8192,0.043765334288279216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,128,0.00589333325624466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,7168,0.019963733355204263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,7168,0.039410134156545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,12288,0.06979093551635743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,6144,0.017233065764109292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,6144,0.034969600041707356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,65536,0.2184885342915853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,5120,0.013876266280810037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,5120,0.030780800183614093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,51200,0.17272000312805175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,4096,0.010775466759999592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,4096,0.025868799289067584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,16384,0.059894398848215735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,3584,0.010398933291435241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,3584,0.025226666529973345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,12288,0.04554239908854167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,3072,0.008750933408737182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,3072,0.023509333531061806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,10240,0.040091733137766525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,2560,0.008154666423797608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,2560,0.021772799889246623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,8192,0.030921600262324017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,2048,0.006930133203665416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,2048,0.020516266425450645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,7168,0.02784213423728943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,1536,0.005906133353710175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,1536,0.018891733884811402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,6144,0.024856533606847128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,1024,0.004837333162625631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,1024,0.017729065815607705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,5120,0.0213536004225413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,768,0.004433066646258036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,768,0.017422932386398315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,4096,0.017569067080815633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,512,0.0039893334110577905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,512,0.016394666830698647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,3584,0.016637866695721946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,256,0.003519999980926514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,256,0.015371732910474143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,3072,0.014648532867431641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,128,0.003239466746648153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,128,0.015072000026702882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,2048,0.011273599664370219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,64,0.003102933367093404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,64,0.015457066893577575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,32,0.003272533416748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,32,0.014990933736165366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,65536,0.06872853438059488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,2560,0.013148799538612366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,1024,0.00802346666653951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,65536,0.29022080103556314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,51200,0.05516586701075236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,51200,0.23062613805135093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,1536,0.009900800387064616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,16384,0.022729599475860597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,16384,0.08382933139801026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,512,0.007163733243942261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,12288,0.018521600961685182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,768,0.007773866752783458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,12288,0.06477760076522827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,10240,0.016302933295567833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,10240,0.05314453442891439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,256,0.006293333570162455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,8192,0.01416106621424357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,8192,0.04104746580123901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,128,0.005789866546789805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,7168,0.012974933783213297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,7168,0.03588159879048665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,6144,0.013808000087738036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,6144,0.03274346590042114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,65536,0.20615572929382325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,5120,0.012272000312805176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,51200,0.1632373332977295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,5120,0.028750934203465778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,4096,0.008957866827646892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,4096,0.02576106588045756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,16384,0.05526613394419352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,3584,0.00837546686331431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,3584,0.025897600253423053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,12288,0.0429418683052063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,3072,0.007956266899903615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,3072,0.023268266518910726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,10240,0.03642559846242269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,2560,0.0075530668099721265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,2560,0.021836799383163453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,8192,0.029934932788213093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,2048,0.0069909334182739254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,7168,0.026206932465235394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,2048,0.01988906661669413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,1536,0.005891199906667074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,1536,0.01962560017903646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,6144,0.022824533780415854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,1024,0.00481386681397756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,1024,0.01795626680056254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,5120,0.019921066363652547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,768,0.004443733394145966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,4096,0.016956800222396852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,768,0.016760534048080443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,512,0.0038890667259693147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,512,0.01643946667512258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,3584,0.015734400351842245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,256,0.003432533393303553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,256,0.015703466534614564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,3072,0.01383786698182424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,128,0.0031669333577156065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,128,0.015019733707110086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,2560,0.012604799866676331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,64,0.002974933385848999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,64,0.015165866414705912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,32,0.0029919999341169994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,32,0.014969600240389505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,65536,0.05888319810231527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,2048,0.010774399836858113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,65536,0.2834176063537598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,51200,0.04752426544825236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,51200,0.22487786610921226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,1536,0.00951039989789327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,16384,0.015123200416564942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,16384,0.07896107037862142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,16384,0.05512533187866211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,12288,0.011316266655921937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,12288,0.05629119873046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,12288,0.042777601877848306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,10240,0.010619733730951946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,512,0.006910933554172516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,10240,0.04635200103123983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,768,0.007292800148328145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,10240,0.036187732219696046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,8192,0.00863039990266164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,7168,0.01179200013478597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,8192,0.03731946547826131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,7168,0.032935466368993124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,6144,0.00824533353249232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,6144,0.031623466809590654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,256,0.005902933577696482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,5120,0.0085098663965861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,5120,0.02783573269844055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,1024,0.007820799946784973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,4096,0.007750399907430013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,4096,0.025329067309697466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,4096,0.01681386629740397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,3584,0.008319999774297078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,3584,0.02405440012613932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,65536,0.2049077351888021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,3584,0.015809067090352378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,128,0.0055978665749231975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,3072,0.023486934105555215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,2560,0.006435200075308482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,2560,0.020883200565973918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,51200,0.16107947031656902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,2048,0.005694933235645294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,2048,0.020115200678507486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,2048,0.010839466253916423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,1536,0.004971733192602793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,1536,0.01836479902267456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,8192,0.029290666182835896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,3072,0.007950933277606964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,1024,0.004312533140182495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,1536,0.009698133667310078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,7168,0.025821866591771443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,1024,0.007937066753705342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,768,0.003914666672547659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,1024,0.01748159925142924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,512,0.0036544000109036768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,768,0.00747626672188441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,768,0.016679465770721436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,512,0.006913066903750102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,512,0.01602133313814799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,6144,0.023175466060638427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,256,0.006075733403364817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,256,0.015491200486818948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,128,0.0029109333952267963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,128,0.014659200112024942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,128,0.005525333185990652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,32,0.0029237332443396253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,64,0.01476693352063497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,32,0.01492693324883779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,65536,0.05385173161824545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,51200,0.04326080083847046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,65536,0.28093865712483723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,16384,0.010686933000882467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,51200,0.22218240102132164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,12288,0.009493333101272584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,16384,0.07619413534800211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,12288,0.056133333841959634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,10240,0.008027733365694682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,8192,0.007147733370463054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,10240,0.04500266710917155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,8192,0.03595199982325236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,7168,0.03291093309720357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,6144,0.008414933085441589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,6144,0.029798400402069092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,256,0.0032970666885375976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,5120,0.00764160007238388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,4096,0.0070613334576288865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,5120,0.028193066517512005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,64,0.002796799937884013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,3584,0.006791466474533081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,3072,0.013803733388582864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,4096,0.025235199928283693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,3072,0.006467199822266896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,2560,0.006295466423034668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,3584,0.023337600628534953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,3072,0.022124799092610677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,2048,0.005514666438102722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,2560,0.021201066176096597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,1536,0.004857600231965383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,2048,0.019629865884780884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,1024,0.004186666508515676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,2560,0.01253973344961802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,768,0.003714133302370707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,1024,0.017468800147374473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,768,0.01678826610247294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,512,0.0034517332911491393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,256,0.003160533308982849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,512,0.015845333536465965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,128,0.002898133297761281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,256,0.016004266341527303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,64,0.0029130667448043824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,128,0.014958932995796204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,64,0.014862933754920959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,32,0.014477866888046264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,65536,0.05165226856867472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,5120,0.01988160014152527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,51200,0.04184746742248535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,16384,0.009758933385213216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,12288,0.008758399883906047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,16384,0.07646719614664713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,10240,0.00811520020167033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,12288,0.05339733362197876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,8192,0.007141333321730297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,7168,0.006899199883143107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,8192,0.03410240014394124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,6144,0.006153599917888641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,7168,0.03205333352088928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,7168,0.0069365332523981735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,5120,0.006493866443634033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,6144,0.02942613363265991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,4096,0.0061141331990559895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,5120,0.027909332513809205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,3584,0.006468266745408376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,4096,0.02518613338470459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,3072,0.006161066889762879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,3584,0.02379306753476461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,3072,0.022791467110315957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,2048,0.005383466680844625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,2560,0.02190613349278768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,1536,0.004790399968624115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,2048,0.01958079934120178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,1024,0.0040618665516376495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,1024,0.01727466583251953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,768,0.0037077332536379496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,768,0.016667733589808144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,512,0.003386666625738144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,512,0.01607039968172709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,32,0.0030634666482607523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,256,0.0030805334448814393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,256,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,128,0.002882133424282074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,128,0.0150218665599823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,64,0.002738133321205775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,65536,0.27843198776245115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,64,0.01492800017197927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,32,0.002696533252795537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,10240,0.04338879982630412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,32,0.014842666188875833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,2560,0.006131199995676676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,1536,0.018268799781799315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,1536,0.018598399559656777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,51200,0.22151360511779786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,16384,1.1926474253336588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,16384,2.249807993570964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,12288,1.6165184020996093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,12288,0.9307338714599609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,10240,0.7409834543863932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,10240,1.4391082763671874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,51200,4.06026128133138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,8192,0.5961429595947265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,8192,1.1928117116292318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,7168,1.06561279296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,7168,0.5246965408325195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,6144,0.8655583699544271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,51200,7.088083394368489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,6144,0.5209727923075358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,5120,0.7334165573120117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,5120,0.4174048105875651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,10240,1.4328192392985026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,12288,1.7079626719156902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,16384,2.5071200052897136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,4096,0.5975392023722331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,4096,0.3214090665181478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,3584,0.5278133392333985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,3584,0.2833813349405924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,3072,0.43852373758951824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,3072,0.2725578625996908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,7168,1.0419296264648437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,8192,1.186786142985026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,2560,0.3515754699707031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,2560,0.2125845273335775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,2048,0.280625057220459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,2048,0.17358187039693196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,6144,0.8785696029663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,5120,0.7347893397013346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,1536,0.14097706476847333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,1536,0.217523193359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,1024,0.1537600040435791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,51200,10.816328938802084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,1024,0.10642773310343426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,768,0.11757013003031414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,768,0.0899072011311849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,4096,0.5873696009318035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,512,0.08974719842274984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,512,0.07509439786275228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,3584,0.5141183853149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,256,0.06574506759643554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,256,0.06403093338012696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,2560,0.3705098787943522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,128,0.05486186742782593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,128,0.06034666697184245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,3072,0.44034665425618486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,64,0.0512661337852478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,64,0.0594592014948527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,65536,32,0.0464138666788737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,65536,32,0.059154133001963295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,2048,0.29903039932250974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,768,0.1348639965057373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,1536,0.22494400342305504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,1024,0.14950505892435711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,256,0.08950719833374024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,128,0.0839413324991862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,65536,512,0.10562773545583089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,65536,3.8792350769042967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,16384,0.9593685150146485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,16384,1.8306315104166668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,51200,3.1877461751302083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,12288,0.7185418446858723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,12288,1.3880160013834635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,65536,7.2668309529622395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,10240,0.5923242568969727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,51200,5.723445129394531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,10240,1.13984743754069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,8192,0.49492588043212893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,7168,0.8206293106079101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,8192,0.8590944290161133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,7168,0.443617057800293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,6144,0.6813525517781576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,6144,0.39268693923950193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,16384,1.9082538604736328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,10240,1.1498826344807944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,12288,1.3341290791829428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,5120,0.5840405146280925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,5120,0.3224149386088053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,4096,0.2614485263824463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,4096,0.45681066513061525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,3584,0.4098656018575032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,8192,0.9405450820922852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,51200,7.986087544759114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,7168,0.8181962966918945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,3584,0.2300704002380371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,3072,0.33508691787719724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,3072,0.20050346056620277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,2560,0.27648000717163085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,2560,0.17231465975443522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,65536,10.826203409830729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,2048,0.22284693717956544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,2048,0.14356266657511393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,1536,0.17324053446451823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,6144,0.6960213343302409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,1536,0.11273066997528076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,1024,0.11950613657633465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,1024,0.08583679993947348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,5120,0.5799573262532551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,768,0.09774186611175537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,768,0.07560746669769287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,4096,0.46431786219278975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,512,0.07093866666158041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,512,0.06346880197525025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,2560,0.29367361068725584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,256,0.04900053342183431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,256,0.056944000720977786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,3584,0.4022325197855632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,128,0.05013866821924845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,128,0.05117119948069254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,1536,0.17849173545837402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,64,0.03837546507517497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,64,0.05146239995956421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,51200,32,0.037749334176381426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,51200,32,0.050876800219217935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,2048,0.2364895979563395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,3072,0.3473962783813477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,1024,0.11962133248647053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,65536,1.3053226470947266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,768,0.10861226717631023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,65536,2.4060267130533854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,51200,1.848366928100586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,256,0.07245866457621256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,512,0.0846933364868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,51200,1.0385791778564453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,16384,0.5419914881388347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,16384,0.35433600743611654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,12288,0.4061429341634114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,12288,0.2644714673360189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,10240,0.2234421412150065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,10240,0.3432959874471029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,8192,0.1832863966623942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,8192,0.2751935958862305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,51200,128,0.0679626703262329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,7168,0.25418987274169924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,7168,0.15749653180440265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,6144,0.20973119735717774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,6144,0.13982826868693035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,10240,0.3632234573364258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,5120,0.1715434710184733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,5120,0.11643733183542888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,12288,0.424450143178304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,51200,2.3081044514973956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,4096,0.14300799369812012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,4096,0.10582933425903321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,65536,2.9467615763346355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,3584,0.12514986991882324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,3584,0.09665066401163737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,16384,0.5805973052978516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,3072,0.1128490686416626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,3072,0.07864533265431722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,2560,0.09093226591746012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,7168,0.2629162629445394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,2560,0.07011840343475342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,2048,0.07485546271006266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,2048,0.061610666910807285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,8192,0.3034346580505371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,1536,0.0577888011932373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,1536,0.050887465476989746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,6144,0.22687892913818358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,1024,0.04271786610285441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,1024,0.04230506817499797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,5120,0.1911413351694743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,768,0.03482773303985596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,768,0.037275731563568115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,4096,0.1527957280476888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,512,0.02618559996287028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,2048,0.07968640327453613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,512,0.03253440062204997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,256,0.018071466684341432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,256,0.028433066606521607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,3584,0.1326485315958659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,128,0.01393066644668579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,128,0.028541866938273115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,2560,0.09756373564402263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,64,0.013591466347376504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,64,0.028467200199762982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,16384,32,0.013635200262069703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,16384,32,0.02753173311551412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,1536,0.05960319836934408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,1024,0.04299626747767131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,65536,1.754412841796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,3072,0.11477440198262531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,65536,1.006985600789388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,51200,1.3660789489746095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,51200,0.7893301645914714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,16384,0.42696533203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,128,0.024778666098912557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,256,0.02684906721115112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,16384,0.27711785634358727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,768,0.03919680118560791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,12288,0.3047616004943848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,12288,0.20426026980082193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,10240,0.25553599993387854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,10240,0.17010560035705566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,16384,512,0.03178453246752421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,8192,0.20189332962036133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,8192,0.138265593846639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,7168,0.17791679700215657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,7168,0.1230463981628418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,6144,0.15556480089823405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,6144,0.10831786791483562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,12288,0.35130132039388023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,5120,0.12929173310597736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,5120,0.09188053607940674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,65536,2.342060852050781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,4096,0.11380053361256917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,4096,0.0860053300857544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,6144,0.18662506739298504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,16384,0.46909440358479815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,3584,0.09325973192850748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,3584,0.06993066469828288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,3072,0.08366080125172934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,10240,0.2982048034667969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,3072,0.06520853439966837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,2560,0.06848959922790528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,2560,0.05767360130945841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,7168,0.2173781394958496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,2048,0.055080533027648926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,2048,0.048438398043314616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,51200,1.7383861541748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,1536,0.05120000044504801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,8192,0.2513397375742594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,1536,0.043882668018341064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,1024,0.03361279964447021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,1024,0.03574719826380412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,3072,0.09323626359303792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,768,0.025217066208521526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,768,0.030437332391738892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,4096,0.12516480286916096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,512,0.019591466585795084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,512,0.028808534145355225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,1536,0.04957973162333171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,256,0.015974400440851848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,256,0.025783467292785644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,5120,0.15636693636576335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,128,0.012843733032544455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,128,0.02442453304926554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,2560,0.07794666290283203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,64,0.011055999994277954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,64,0.02478613257408142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,12288,32,0.011348266402880352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,12288,32,0.02462293306986491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,3584,0.10935253302256268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,1024,0.03678293228149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,2048,0.06275093158086141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,65536,1.4085088094075522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,65536,0.9099242528279623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,51200,1.1113707224527993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,16384,0.36183039347330725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,16384,0.23810985883076988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,16384,0.40103680292765304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,12288,0.258790397644043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,12288,0.17758933703104657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,51200,0.7232255935668945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,10240,0.217413330078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,10240,0.1490122636159261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,128,0.015799466768900552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,8192,0.17387946446736652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,512,0.022320000330607097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,8192,0.12114026546478271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,8192,0.2115605354309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,7168,0.16680320103963214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,7168,0.11767146587371827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,6144,0.14647787412007648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,6144,0.10517226854960124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,256,0.018206934134165444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,65536,1.959058125813802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,12288,768,0.02781333327293396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,5120,0.12001813252766927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,5120,0.09844480355580648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,4096,0.09714346726735433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,51200,1.4426986694335937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,4096,0.07897280057271322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,3584,0.06992639700571696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,3584,0.09871466954549155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,3072,0.0825045347213745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,3072,0.06579840183258057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,3072,0.07975680033365885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,2560,0.06989013353983561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,2560,0.058405331770579015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,12288,0.30084587732950846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,10240,0.25394986470540365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,2048,0.0522325317064921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,2048,0.045688533782958986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,7168,0.18525226910909015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,1536,0.03981866836547852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,1536,0.03864746491114299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,1536,0.04359680016835531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,1024,0.028318933645884198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,1024,0.030755199988683063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,6144,0.15892693201700847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,768,0.022817067305246987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,768,0.029072000583012896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,5120,0.13051733175913494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,512,0.016535466909408568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,4096,0.10891199906667073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,512,0.027276800076166792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,256,0.01243839959303538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,256,0.023809067408243813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,3584,0.09351680278778077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,128,0.01092693308989207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,128,0.02111466725667318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,128,0.013672533631324767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,64,0.009779199957847595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,64,0.02212266723314921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,10240,32,0.010175999999046326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,10240,32,0.022089600563049316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,2560,0.06674986680348714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,2048,0.054744533697764074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,65536,0.7344426472981771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,65536,1.1439520517985025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,51200,0.5735392252604167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,51200,0.8581109364827475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,16384,0.26670506795247395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,1024,0.03158719937006633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,768,0.02475093404452006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,16384,0.2103498617808024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,12288,0.19803306261698406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,512,0.01919680039087931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,12288,0.14320319493611652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,10240,0.16606613794962566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,10240,256,0.015889066457748412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,8192,0.13537813822428385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,8192,0.09888746738433837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,7168,0.11930987040201824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,7168,0.08901013533274332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,10240,0.11941653092702229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,6144,0.10139626661936443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,6144,0.07847253481547037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,12288,0.23952320416768394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,5120,0.0856661319732666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,5120,0.06925439834594727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,65536,1.528170649210612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,4096,0.07271146774291992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,4096,0.06341333389282226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,16384,0.32890345255533854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,3584,0.06234773397445679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,10240,0.20820266405741372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,3584,0.05357760190963745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,3072,0.053852800528208414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,3072,0.047787733872731525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,6144,0.13318400382995604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,2560,0.04528426726659139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,2560,0.041842134793599446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,7168,0.1521898587544759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,2048,0.03725013335545858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,2048,0.03695679903030395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,51200,1.1384639739990234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,1536,0.029582933584849043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,1536,0.0325984001159668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,8192,0.17756692568461102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,1024,0.021090133984883627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,1024,0.027356799443562823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,3072,0.06790186564127604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,768,0.01718719998995463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,768,0.025172267357508344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,1536,0.0370527982711792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,512,0.013579733173052468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,512,0.022632533311843873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,4096,0.08877867062886556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,256,0.009493333101272584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,256,0.02020906607309977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,5120,0.10972266991933186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,128,0.008002133170763651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,2560,0.055622398853302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,128,0.019514666001001994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,64,0.006909866631031036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,64,0.019604265689849854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,8192,32,0.007358933488527934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,8192,32,0.01972800095876058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,3584,0.07957013448079428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,1024,0.026412800947825117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,2048,0.04591786861419678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,65536,1.0133546829223632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,65536,0.7103413263956706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,51200,0.5745546976725261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,51200,0.7912991841634114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,16384,0.2523178736368815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,16384,0.2031925360361735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,12288,0.19550293286641438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,12288,0.15224533081054686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,256,0.013479466239611307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,128,0.011573333541552227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,10240,0.15954987208048504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,10240,0.12035199801127117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,512,0.016806399822235106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,8192,0.12833600044250487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,8192,0.10106133619944255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,8192,768,0.02146986722946167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,7168,0.11411519845326741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,7168,0.09203519821166992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,6144,0.09939733346303305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,6144,0.08170026938120524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,51200,0.8584672292073569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,5120,0.08880746364593506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,5120,0.07362346649169922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,65536,1.1357215881347655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,4096,0.06876266797383626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,4096,0.058221864700317386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,16384,0.2630261262257894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,3584,0.06138453483581543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,3584,0.051577599843343105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,10240,0.17255466779073078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,3072,0.053506131966908774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,12288,0.1970069408416748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,3072,0.046507732073465986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,2560,0.04554773171742757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,2560,0.04041279951731364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,7168,0.12292053699493408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,2048,0.036084266503651936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,2048,0.03493760029474895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,8192,0.14140052795410157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,1536,0.027857067187627156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,1536,0.03123520016670227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,6144,0.10548053582509358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,1024,0.019426133235295615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,1024,0.027373866240183516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,5120,0.0891103982925415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,768,0.01569066643714905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,768,0.02482453385988871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,4096,0.07241386572519938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,512,0.011832533280054729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,512,0.02294506629308065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,2560,0.048819200197855635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,256,0.009262933333714803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,256,0.019811199108759562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,3584,0.06274986664454142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,128,0.007405866682529449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,128,0.0190175990263621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,2048,0.03807466824849447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,64,0.006514133512973785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,64,0.019337600469589232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,7168,32,0.006818133095900218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,7168,32,0.019086933135986327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,65536,0.8115168253580729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,3072,0.054295468330383304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,1536,0.029983999331792195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,65536,0.6047957102457683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,768,0.020016000668207804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,51200,0.6213151931762695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,51200,0.47847894032796223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,16384,0.20154879887898763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,16384,0.16175039609273273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,1024,0.02254293362299601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,12288,0.15221439997355143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,256,0.013967999815940857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,12288,0.11716266473134358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,512,0.016782933473587038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,12288,0.18371520042419434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,10240,0.12591466903686524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,10240,0.10042453606923421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,8192,0.10325013001759846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,8192,0.08355200290679932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,7168,0.09084373315175374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,7168,0.07517653306325277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,6144,0.07744959990183511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,6144,0.06729386647542318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,7168,128,0.012971733013788858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,5120,0.0652181347211202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,5120,0.058988801638285314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,5120,0.08455253442128499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,4096,0.05313706795374552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,4096,0.05045226812362671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,7168,0.11621013482411702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,3584,0.04745066563288371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,3584,0.04610133171081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,65536,1.0821706136067708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,3072,0.04250346819559733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,3072,0.04299626747767131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,3072,0.05106133222579956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,2560,0.035359998544057206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,51200,0.7963285446166992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,16384,0.2558805306752523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,2560,0.03749866485595703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,2048,0.028936533133188884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,2048,0.033589335282643636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,1536,0.023754666248957314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,1536,0.02911146680514018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,10240,0.1611626625061035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,1024,0.01741546591122945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,8192,0.1351637363433838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,1024,0.02526613275210063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,6144,0.10357333024342855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,768,0.014408533771832785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,768,0.02265066703160604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,4096,0.068067200978597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,512,0.011482666929562886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,512,0.022201599677403767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,3584,0.059638400872548424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,256,0.009398399790128072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,256,0.019234132766723634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,2560,0.04328426520029704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,128,0.006934399902820587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,128,0.018346667289733887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,2048,0.03520319859186809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,64,0.006044800082842508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,1536,0.02832213242848714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,6144,32,0.006658133367697399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,64,0.018677333990732826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,6144,32,0.018218666315078735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,1024,0.02078933318456014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,65536,0.5348384221394856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,65536,0.71156800587972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,768,0.017998933792114258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,16384,0.17390079498291017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,51200,0.5267519950866699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,51200,0.4346474647521973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,16384,0.14155306816101074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,12288,0.13204267024993896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,512,0.013479466239611307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,12288,0.10738346576690674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,10240,0.10964907010396321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,10240,0.09228800137837728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,8192,0.08958933353424073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,8192,0.07703680197397868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,256,0.011287466684977213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,7168,0.07840960025787354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,7168,0.06985812981923421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,6144,128,0.009959466258684794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,6144,0.06886719862620036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,6144,0.06331413189570109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,12288,0.17245333989461262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,5120,0.05738879839579264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,5120,0.054730665683746335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,65536,0.9966997146606446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,4096,0.04911893208821615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,4096,0.04864960114161174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,16384,0.23233599662780763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,3584,0.04070613384246826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,3584,0.04282346566518148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,10240,0.14668265978495282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,3072,0.03594026565551758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,3072,0.03879573345184326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,6144,0.09409600098927816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,2560,0.031352533896764116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,2560,0.03452479839324951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,51200,0.7489952087402344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,2048,0.026625066995620728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,2048,0.031411200761795044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,7168,0.10694399674733479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,1536,0.02071573336919149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,1536,0.027508266766866046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,8192,0.12547093232472736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,1024,0.01565120021502177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,1024,0.024420267343521117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,3072,0.04942613442738851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,768,0.013289599617322286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,768,0.021939200162887574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,1536,0.0271178662776947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,512,0.010428800185521444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,512,0.021554134289423623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,4096,0.06427733500798544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,256,0.007828266421953837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,2560,0.0408405343691508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,256,0.018528000513712565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,128,0.006105599800745646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,128,0.0180565337340037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,5120,0.07797119617462159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,64,0.00565119981765747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,64,0.01800959904988607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,5120,32,0.006054399907588959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,5120,32,0.018148267269134523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,3584,0.05526506503423055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,65536,0.5231328010559082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,65536,0.44918187459309894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,1024,0.019428267081578573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,2048,0.03371520042419433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,51200,0.40138559341430663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,51200,0.3585834821065267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,16384,0.1454261302947998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,16384,0.11760853131612141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,12288,0.10054399967193603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,12288,0.09047786394755045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,65536,0.7305610656738282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,16384,0.18127360343933105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,128,0.010007466872533162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,256,0.010985599954922994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,10240,0.08460373083750407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,8192,0.06893866856892904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,51200,0.56156800587972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,10240,0.11770880222320557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,8192,0.06834026972452799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,7168,0.05944960117340088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,12288,0.13079360326131184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,7168,0.06156053145726522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,6144,0.05182079871495565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,6144,0.05489919980367025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,5120,0.04389760096867879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,512,0.013314132889111837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,5120,0.046838398774464926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,4096,0.0356383999188741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,4096,0.03964693148930867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,5120,768,0.01725333333015442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,3584,0.03144213358561198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,3584,0.035818668206532796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,10240,0.07756693363189697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,3072,0.027169066667556762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,3072,0.03282879988352458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,3072,0.038311465581258135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,2560,0.02324906587600708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,2560,0.029864533742268877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,2048,0.01963520050048828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,8192,0.10280213356018067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,2048,0.02681279977162679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,7168,0.08587199846903483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,1536,0.015863466262817382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,6144,0.0741429328918457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,1024,0.011934933066368104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,1536,0.02518933415412903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,1024,0.021010132630666097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,5120,0.06047573486963907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,768,0.010052266716957092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,768,0.020075732469558717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,512,0.008124800026416778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,512,0.01879146695137024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,4096,0.04909653266270955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,256,0.006138666470845541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,256,0.01764799952507019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,256,0.00902826686700185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,128,0.005351466437180838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,128,0.01695893406867981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,3584,0.04405226707458496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,128,0.008060800035794576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,64,0.004869333406289419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,4096,32,0.005047466854254405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,64,0.01707520087560018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,4096,32,0.017173333962758382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,2560,0.031660799185434976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,65536,0.4343253453572591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,51200,0.36937173207600915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,51200,0.3494943936665853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,65536,0.4773258527119954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,51200,0.5220128059387207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,2048,0.025645865996678667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,16384,0.12098133563995361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,1536,0.022181334098180135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,16384,0.11523733139038086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,12288,0.091975466410319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,1024,0.01593706707159678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,10240,0.07739520072937012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,12288,0.08622506459554037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,768,0.014030933380126953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,10240,0.07463040351867675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,8192,0.061332265535990395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,10240,0.11025066375732422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,8192,0.06359039942423503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,4096,512,0.010681600371996561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,7168,0.05450133482615153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,6144,0.04763199885686238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,7168,0.057555198669433594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,7168,0.08043413162231446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,6144,0.05154240131378174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,5120,0.04011840025583903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,5120,0.045628801981608076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,4096,0.03280426661173503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,4096,0.03754239877065023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,4096,0.0476032018661499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,3584,0.029363199075063066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,3584,0.03460373481114705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,3072,0.0255295991897583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,3584,0.04259200096130371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,3072,0.03168533245722453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,2560,0.021691733598709108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,2560,0.02890026569366455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,65536,0.6806133270263672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,2048,0.018205867211023966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,2048,0.02704746723175049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,16384,0.17265067100524903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,1536,0.01506239970525106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,12288,0.12857386271158855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,1536,0.023875200748443605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,1024,0.01159999966621399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,1024,0.021427200237909953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,8192,0.09372693697611491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,768,0.009640533725420635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,6144,0.07007359663645427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,768,0.020105600357055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,512,0.007715199887752533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,5120,0.06052800019582113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,512,0.018713599443435668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,256,0.006098133325576782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,3072,0.037137067317962645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,256,0.017565866311391197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,2560,0.030426667133967085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,128,0.005227733155091604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,128,0.01694933374722799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,2048,0.02600640058517456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,64,0.00470719983180364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,64,0.01714986761411031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3584,32,0.005017599960168203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3584,32,0.01723946730295817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,65536,0.4111925443013509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,1536,0.02092906634012858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,1024,0.014919466773668923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,65536,0.41097278594970704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,768,0.013722667098045349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,512,0.010615467031796774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,51200,0.33530985514322914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,51200,0.3264458656311035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,16384,0.10349546273549397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,12288,0.08100053469340006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,16384,0.1063968022664388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,12288,0.08051413695017497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,10240,0.0698752005894979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,10240,0.07001279989878337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,256,0.009012266993522644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,8192,0.05509119828542074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3584,128,0.008222933113574981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,7168,0.050185600916544594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,7168,0.05430506865183512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,8192,0.05975679953893026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,6144,0.04375360012054443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,6144,0.04790613253911336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,12288,0.11977919737497966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,5120,0.034907734394073485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,5120,0.04164479970932007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,16384,0.16309866905212403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,65536,0.6251904169718425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,4096,0.03028800090154012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,4096,0.03602666854858398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,3584,0.025265065828959148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,3584,0.03392639954884847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,10240,0.10478080113728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,3072,0.021977599461873373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,3072,0.03006826639175415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,7168,0.0759722630182902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,2560,0.020068265994389854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,2560,0.027149866024653118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,6144,0.0662015994389852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,2048,0.016241066654523215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,2048,0.025245867172876996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,8192,0.08858773708343506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,1536,0.013123200337092081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,1536,0.02346986730893453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,51200,0.4851893424987793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,1024,0.01023573378721873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,1024,0.020772266387939452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,3072,0.03508266607920329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,768,0.009016533692677815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,768,0.01936960021654765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,4096,0.045067731539408365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,512,0.006930133203665416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,5120,0.05401279926300049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,512,0.017968000968297322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,256,0.005524266759554545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,256,0.017041067282358803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,2560,0.029191466172536214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,128,0.004791466891765595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,3584,0.03929920196533203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,128,0.016411733627319337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,64,0.004339199761549632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,64,0.016755199432373045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,3072,32,0.004702933132648468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,3072,32,0.016697599490483602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,1536,0.019308799505233766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,2048,0.024247467517852783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,65536,0.35778239568074544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,1024,0.014062933127085366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,65536,0.36851412455240884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,51200,0.2899989446004232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,51200,0.29269653956095376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,16384,0.09225707054138184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,16384,0.09580693244934083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,12288,0.0685749371846517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,12288,0.07492693265279135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,256,0.008585600058237712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,10240,0.056279468536376956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,10240,0.06578346490859985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,128,0.007841066519419352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,768,0.012735999623934426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,8192,0.046317867437998456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,3072,512,0.010050132870674133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,8192,0.05601813395818075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,7168,0.041647998491923015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,7168,0.05035200119018555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,6144,0.03527466853459676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,6144,0.04408213297526042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,12288,0.1102666695912679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,5120,0.03030719955762227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,5120,0.038574934005737305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,65536,0.590887451171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,16384,0.15155199368794758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,4096,0.024885332584381102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,4096,0.03312000036239624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,51200,0.4787925402323405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,3584,0.021862399578094483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,3584,0.030554666121800737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,6144,0.06321706771850585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,3072,0.019258666038513183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,3072,0.028602667649586994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,10240,0.09538239638010661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,2560,0.01686933239301046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,2560,0.026220800479253133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,2048,0.01460906664530436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,2048,0.02472320000330607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,7168,0.07147520383199056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,1536,0.012072533369064331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,8192,0.08409066994984946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,1536,0.02257279952367147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,1024,0.009369599819183349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,1024,0.020009599129358926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,5120,0.051369599501291906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,768,0.007740800082683563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,768,0.019035732746124266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,4096,0.041484800974527995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,512,0.006458666423956554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,512,0.017893334229787193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,3584,0.037988265355428055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,256,0.005393066505591074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,256,0.017593600352605186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,3072,0.03280959924062093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,128,0.0046186665693918865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,128,0.0159850666920344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,1536,0.018337066968282065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,64,0.004411733150482178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,64,0.016356266538302102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2560,32,0.0045162667830785114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2560,32,0.01625599960486094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,2560,0.02727466622988383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,65536,0.2703690528869629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,1024,0.013365333278973898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,65536,0.3345130602518717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,65536,0.4171018600463867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,51200,0.21854613622029623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,51200,0.2674367904663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,16384,0.08811093171437581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,16384,0.09721493721008301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,16384,0.11025813420613606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,12288,0.06883412996927897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,12288,0.07778346538543701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,10240,0.058508801460266116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,10240,0.06692799727121988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,10240,0.0691701332728068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,8192,0.04536426862080892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,8192,0.05422720114390055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,2048,0.022294400135676067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,7168,0.04149119853973389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,7168,0.04761600097020467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,7168,0.05080639918645223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,6144,0.03578773339589437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,6144,0.04242666562398274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,6144,0.04368106524149577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,5120,0.03209386666615804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,5120,0.0371509313583374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,5120,0.03697386582692464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,512,0.009545600414276123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,4096,0.024477867285410564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,4096,0.032500267028808594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,3584,0.022232532501220703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,3584,0.02992960015932719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,128,0.007477333148320516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,3072,0.019619200627009073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,256,0.008249600231647492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,3072,0.02773759961128235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,2560,0.01688533425331116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,2560,0.026205867528915405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,2560,0.021254400412241616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,2048,0.014260266224543253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,2048,0.022461867332458495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,2048,0.017280000448226928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,1536,0.011353600025177001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,1536,0.02129813234011332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2560,768,0.012151466806729634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,1024,0.008705066641171773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,1024,0.01939093271891276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,12288,0.08236693541208903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,768,0.00727893312772115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,768,0.018660267194112144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,51200,0.32521174748738607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,512,0.006159999966621399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,3072,0.02336960037549337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,512,0.01747200091679891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,512,0.008071466783682505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,256,0.004852266609668731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,1536,0.013705600301424661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,256,0.01650773286819458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,128,0.004125866790612539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,128,0.015974400440851848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,64,0.0038453333079814913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,64,0.0159061332543691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,2048,32,0.004098133246103922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,8192,0.05843733151753744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,2048,32,0.016241066654523215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,65536,0.21459840138753256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,4096,0.029938133557637532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,65536,0.3084842681884766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,51200,0.1707754611968994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,65536,0.37052478790283205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,51200,0.25126399993896487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,16384,0.07943572998046874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,16384,0.09327466487884521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,3584,0.02691626747449239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,12288,0.061654400825500486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,1024,0.01086293359597524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,12288,0.07491520245869955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,10240,0.06481813192367554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,51200,0.28711252212524413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,128,0.006648533542950948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,8192,0.038932267824808756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,8192,0.050945067405700685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,7168,0.0354144016901652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,16384,0.09862186908721923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,7168,0.04538240035374959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,7168,0.04581973155339559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,6144,0.031122134129206343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,6144,0.03990613222122193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,256,0.0071722666422526045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,5120,0.026922667026519777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,10240,0.0537770668665568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,4096,0.021603200833002725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,5120,0.03577173153559367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,4096,0.03143253326416016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,2048,768,0.009461333354314167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,3584,0.019435733556747437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,3584,0.028475733598073323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,3072,0.016434133052825928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,12288,0.07306026617685954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,3072,0.02709546685218811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,2560,0.013894400000572205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,3072,0.021341866254806517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,2560,0.02491413354873657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,2048,0.012340266505877178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,2048,0.02173973321914673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,10240,0.061887999375661217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,1536,0.00978559950987498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,1536,0.020387200514475505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,1536,0.012194133798281352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,1024,0.007196799914042156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,1024,0.01839039921760559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,8192,0.05101866722106933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,768,0.0061482667922973635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,768,0.017754666010538735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,6144,0.03937386671702067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,512,0.005072000126043955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,5120,0.03297706643740336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,512,0.017267199357350667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,256,0.004243200023969015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,256,0.016160000363985697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,4096,0.02719893256823222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,128,0.00377813329299291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,128,0.015656532843907674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,3584,0.02389226754506429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,64,0.003516799956560135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,64,0.015498666961987814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1536,32,0.0036874666810035707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1536,32,0.015662933389345803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,2048,0.015565866231918335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,65536,0.1401535987854004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,65536,0.2664053281148275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,51200,0.11094079812367756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,51200,0.21293333371480308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,2560,0.01849386692047119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,16384,0.044660266240437826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,16384,0.07523626486460368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,1024,0.010120532910029094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,12288,0.032077866792678836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,12288,0.060957864920298255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,768,0.008675199747085572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,10240,0.027239465713500978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,512,0.007580799857775371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,10240,0.05106666485468546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,8192,0.021689599752426146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,8192,0.04180053472518921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,128,0.0060479998588562015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,7168,0.022501333554585775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,7168,0.037560534477233884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1536,256,0.006628266473611195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,6144,0.019761067628860474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,6144,0.034030934174855546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,65536,0.32342828114827477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,5120,0.01755733291308085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,5120,0.030152533451716108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,51200,0.24172266324361166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,4096,0.014525866508483887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,4096,0.02731093366940816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,16384,0.08357120354970296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,3584,0.013339733084042868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,12288,0.06355306704839071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,3584,0.02539413372675578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,3072,0.012084266543388367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,3072,0.024233599503835045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,10240,0.05453973213831583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,8192,0.04442880153656006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,2560,0.010683733224868774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,2560,0.022632533311843873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,7168,0.03927573362986247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,2048,0.00904960036277771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,2048,0.020798933506011964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,6144,0.03402773141860962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,1536,0.007533866663773854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,1536,0.019688532749811808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,1024,0.006345599889755249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,1024,0.018103466431299845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,5120,0.028731733560562134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,768,0.005638400216897329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,768,0.01773973306020101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,4096,0.024366933107376098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,512,0.004941866795221964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,512,0.016673066218694053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,3072,0.018897066513697304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,256,0.004108799993991852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,256,0.015939199924468996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,2048,0.013731200496355692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,128,0.0037589333951473236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,128,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,3584,0.022001065810521445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,64,0.0034400001168251038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,64,0.01565333306789398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,1024,32,0.003489066660404205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,1024,32,0.015449600418408713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,65536,0.11372160116831462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,2560,0.016357333461443583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,65536,0.25198613802591957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,51200,0.09122239748636882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,51200,0.20013119379679362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,1536,0.011834667126337687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,16384,0.046419199307759604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,1024,0.009086933732032777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,16384,0.0822538693745931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,12288,0.0449290672938029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,12288,0.06275306542714437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,512,0.007099733253320058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,10240,0.037588266531626384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,256,0.006387199958165486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,768,0.008152533570925396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,10240,0.05197013219197592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,8192,0.030083199342091877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,8192,0.043274664878845216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,7168,0.027929600079854327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,7168,0.038098132610321044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,1024,128,0.005842133363087972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,6144,0.023785599072774253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,6144,0.03431253433227539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,65536,0.28115946451822915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,5120,0.02002560098965963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,5120,0.030605866511662798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,51200,0.22381439208984374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,4096,0.016934400796890257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,12288,0.05818666617075602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,4096,0.02645653287569682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,3584,0.014657066265741984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,3584,0.025124265750249224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,16384,0.07693760395050049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,3072,0.012957866986592612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,3072,0.02355626622835795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,10240,0.04937813282012939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,2560,0.01132480005423228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,2560,0.02219840089480082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,7168,0.03583360115687052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,2048,0.009452799956003826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,2048,0.020794665813446044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,8192,0.041041068236033124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,6144,0.031013333797454835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,1536,0.007915733257929485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,1536,0.01979093352953593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,1024,0.00606826643149058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,1024,0.017849600315093993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,5120,0.025621332724889118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,4096,0.021596799294153847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,3072,0.01687893271446228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,768,0.005211733281612396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,512,0.004423466821511587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,768,0.016945066054662068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,512,0.016635732849438985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,256,0.0036746665835380556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,256,0.015641599893569946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,3584,0.01955733299255371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,128,0.0034143999218940735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,128,0.015198933084805808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,1536,0.010258133212725323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,2560,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,64,0.0031360000371932983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,768,32,0.0032672000428040824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,64,0.015215999881426492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,768,32,0.01527253290017446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,65536,0.08893547058105469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,65536,0.2441802660624186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,1024,0.00825386643409729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,51200,0.08927146593729654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,2048,0.01241600016752879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,51200,0.19248959223429363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,16384,0.030907734235127764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,16384,0.06861226558685303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,256,0.0055744002262751256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,512,0.0063285330931345625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,128,0.005268266797065735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,12288,0.053592534859975186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,10240,0.02207466761271159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,10240,0.04496959845225017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,8192,0.016225066781044007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,768,768,0.007201066613197327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,8192,0.03707733154296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,7168,0.018523732821146645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,7168,0.03333866596221924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,12288,0.02432960073153178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,6144,0.015065600474675497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,6144,0.029611732562383013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,65536,0.27035306294759115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,5120,0.013025066256523133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,5120,0.026867200930913288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,51200,0.22428159713745116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,4096,0.013860266407330832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,4096,0.02533973256746928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,16384,0.07273279825846354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,3584,0.011450666189193725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,3584,0.02300800085067749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,12288,0.05524266560872396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,3072,0.010196266571680705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,3072,0.021985065937042237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,10240,0.04643306732177734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,2560,0.00832533339659373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,2560,0.02108479936917623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,8192,0.03881173531214396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,2048,0.007298133273919423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,2048,0.01955946683883667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,7168,0.034482133388519284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,1536,0.0061941335598627726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,1536,0.018538665771484376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,6144,0.02927359938621521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,1024,0.0051136001944541935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,1024,0.01738986571629842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,5120,0.02487466732660929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,768,0.004680533210436503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,768,0.017012266318003337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,4096,0.02048106590906779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,512,0.004035199930270513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,512,0.016216533382733662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,3584,0.01874666611353556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,256,0.003551999976237615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,256,0.01569493313630422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,3072,0.016481066743532814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,128,0.003190399954716364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,2048,0.012504532933235168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,128,0.015396266182263692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,64,0.003011200080315272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,64,0.015046399831771851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,512,32,0.003092266619205475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,512,32,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,65536,0.055961600939432775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,2560,0.014705066879590353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,65536,0.22428587277730308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,51200,0.045296001434326175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,1536,0.009656533598899841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,1024,0.0077450667818387345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,51200,0.18166933059692383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,16384,0.02101866602897644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,768,0.007046400010585785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,12288,0.016687999169031777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,16384,0.06391253471374511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,12288,0.046573865413665774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,10240,0.014866133530934652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,10240,0.039341866970062256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,512,0.006229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,8192,0.012478933731714884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,8192,0.03343679904937744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,256,0.0056415999929110205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,7168,0.012121599912643433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,7168,0.03052053252855937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,512,128,0.005224533379077911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,6144,0.010990933577219645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,6144,0.02820906639099121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,65536,0.25436693827311196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,5120,0.011399466792742412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,5120,0.02568746606508891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,51200,0.19578453699747722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,4096,0.008917333682378133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,4096,0.02455893357594808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,16384,0.06789440313975016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,3584,0.008027733365694682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,12288,0.05127360026041666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,10240,0.04338239828745524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,3072,0.007660800218582153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,3072,0.02119893431663513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,8192,0.035359998544057206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,2560,0.0067456002036730455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,2560,0.019398399194081626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,3584,0.022421334187189737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,2048,0.005960533519585928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,2048,0.01895786722501119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,7168,0.03226240078608195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,1536,0.005226666728655497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,6144,0.02783573269844055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,1536,0.018141865730285645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,5120,0.024149332443873087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,1024,0.004439466694990794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,1024,0.01709653337796529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,3072,0.015424000223477683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,768,0.004154666761557261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,4096,0.019284266233444213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,768,0.016851200660069784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,512,0.0037087999284267426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,512,0.01580693324406942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,256,0.0032757334411144257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,256,0.015084800124168397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,3584,0.01732053359349569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,128,0.003102933367093404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,128,0.014866133530934652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,2048,0.011229866743087768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,64,0.0028768000503381092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,64,0.015553067127863566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,2560,0.01359999974568685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,256,32,0.0029365333418051405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,256,32,0.01471466620763143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,65536,0.04657173156738281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,65536,0.21660586992899575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,51200,0.0378602663675944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,51200,0.17263894081115722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,1536,0.0092330664396286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,16384,0.012195199728012085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,16384,0.05802133480707804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,1024,0.007653333246707916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,12288,0.011370666821797689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,12288,0.04159786701202393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,512,0.005985066791375478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,10240,0.010819199681282043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,256,0.0053247998158137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,10240,0.03770346641540527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,8192,0.010327466328938802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,768,0.006648533542950948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,256,128,0.004977066814899444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,8192,0.03165439963340759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,7168,0.009578667084376017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,7168,0.030308266480763752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,6144,0.008567466338475545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,6144,0.027642667293548584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,65536,0.18723093668619792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,5120,0.00986240009466807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,5120,0.025569067398707075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,51200,0.14341972668965658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,4096,0.0085098663965861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,4096,0.023104000091552734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,16384,0.05095146497090658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,3584,0.007911466558774312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,3584,0.022425599892934165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,12288,0.04021120071411133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,3072,0.007031466563542683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,3072,0.020619734128316244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,10240,0.032986666758855185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,2560,0.006355200211207073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,2560,0.020756266514460244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,8192,0.027026132742563887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,2048,0.005634133517742157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,2048,0.01876266598701477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,7168,0.023875200748443605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,1536,0.004920533299446106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,6144,0.02111146648724874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,1536,0.01827733318010966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,1024,0.0042805333932240805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,1024,0.016966400543848674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,5120,0.018326399723688762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,768,0.0038783999780813852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,768,0.016343466440836587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,4096,0.015190399686495461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,512,0.0035189333061377203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,512,0.015717333555221556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,3072,0.012888532876968384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,3584,0.014219733079274497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,256,0.0032586666444937387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,256,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,128,0.0029877332349618276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,128,0.015016532937685647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,2048,0.009810133775075277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,64,0.0027850667635599775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,64,0.014856533209482829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,1024,0.0074314668774604796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,128,32,0.002889599899450938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,128,32,0.014686933159828186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,65536,0.04287573496500651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,51200,0.034881067276000974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,1536,0.008687999844551087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,16384,0.011648000280062357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,2560,0.011684266726175944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,65536,0.21522879600524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,16384,0.055485868453979494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,12288,0.008808533350626629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,10240,0.008181333541870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,12288,0.04101440111796061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,8192,0.007366399963696797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,10240,0.034302934010823564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,8192,0.030506666501363116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,7168,0.008754133184750875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,7168,0.02837439974149068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,6144,0.026753065983454387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,5120,0.008387200037638346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,5120,0.02547093431154887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,4096,0.0071829333901405334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,4096,0.023396267493565878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,3584,0.0066655998428662615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,3584,0.02243306636810303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,3072,0.00639466643333435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,3072,0.021027199427286782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,2560,0.006140799820423126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,2560,0.01946880022684733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,2048,0.005393066505591074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,2048,0.019709867238998414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,1536,0.0047541335225105286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,1536,0.017933867375055947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,1024,0.0040618665516376495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,51200,0.1709994633992513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,1024,0.01695573329925537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,768,0.003869866579771042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,768,0.016532267133394875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,512,0.015571199854214988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,6144,0.008780800302823384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,768,0.00653653343518575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,256,0.00317546675602595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,128,0.002998399982849757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,128,0.014710399508476257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,64,0.002867199977238973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,256,0.014793599645296732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,32,0.0028309332827727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,64,0.015034666657447815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,64,32,0.014709333578745524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,65536,0.04461119969685872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,51200,0.03586346705754598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,512,0.005809066692988077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,16384,0.01165120005607605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,12288,0.009673600395520527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,16384,0.054530131816864016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,65536,0.2145909309387207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,12288,0.03844906489054362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,51200,0.17090560595194498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,8192,0.008332799871762593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,7168,0.00793280005455017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,10240,0.03355413277943929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,6144,0.006492800017197926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,7168,0.02867199977238973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,6144,0.02656960090001424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,5120,0.00730560024579366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,4096,0.0070613334576288865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,5120,0.025175466140111284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,3584,0.006701866785685222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,4096,0.023082667589187623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,3072,0.006470400094985962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,3584,0.02230186661084493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,2560,0.006298666695753734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,3072,0.020890667041142782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,2048,0.005457066496213277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,2048,0.01904426614443461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,1536,0.004796800017356872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,64,512,0.003537066777547201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,1536,0.01755839983622233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,1024,0.004065066576004028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,768,0.0037130666275819145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,1024,0.016668800512949625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,512,0.0033749334514141084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,768,0.016499200463294984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,256,0.003052799900372823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,512,0.01590079963207245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,128,0.0028213332096735638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,256,0.014979199568430582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,64,0.0026922665536403658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,64,0.014935466647148132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,32,0.0026858667532602947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,32,0.014516266187032065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,769,32,10240,0.008916266759236654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,8192,0.031155200799306233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,256,0.0053610667586326596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,769,128,128,0.004977066814899444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,2560,0.019606399536132812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,769,32,128,0.014837333559989929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,16384,1.1481248219807942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,12288,0.8909311930338542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,12288,1.6432586669921876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,16384,2.221155293782552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,10240,0.7557717641194661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,10240,1.4915392557779947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,51200,4.003684234619141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,8192,0.6000949223836263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,16384,1.7091519673665363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,7168,1.0153280258178712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,8192,1.1576895395914713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,7168,0.5777365366617839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,12288,1.247593561808268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,6144,0.7963093439737956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,10240,1.060115178426107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,6144,0.4652906735738118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,51200,7.095604451497396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,5120,0.41901973088582356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,5120,0.5280810674031575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,4096,0.3189141273498535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,4096,0.5509002685546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,3584,0.4871061325073242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,8192,0.8118730545043945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,3584,0.280837345123291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,5120,0.7068576176961263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,3072,0.23875199953715004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,3072,0.40923519134521485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,3584,0.37008854548136394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,2560,0.22134186426798502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,2560,0.3920031865437826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,7168,0.7244373321533203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,2048,0.29866345723470056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,6144,0.6310901641845703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,2048,0.1930858612060547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,1536,0.2358400026957194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,1536,0.14653867085774738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,1024,0.15134612719217938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,1024,0.10649279753367107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,51200,7.486838277180989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,4096,0.41926291783650715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,768,0.12203093369801839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,768,0.0893951972325643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,512,0.07307946681976318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,512,0.0861834685007731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,256,0.05806080102920532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,256,0.06456853151321411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,3072,0.31529067357381185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,128,0.050697600841522215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,2560,0.27168426513671873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,128,0.05899200042088827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,64,0.04522986809412639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,64,0.059799468517303465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,32,0.045355733235677084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,32,0.05910186767578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,2048,0.2057546615600586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,1536,0.16692694028218585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,1024,0.12871466477711996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,768,0.11767253081003826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,512,0.09510293006896972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,256,0.07979413668314615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,128,0.07468266487121582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,16384,1.8366580963134767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,16384,0.9471594492594401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,51200,3.107591501871745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,65536,3.8928148905436197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,12288,1.4117056528727212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,12288,0.6780405044555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,12288,1.014460817972819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,10240,0.5761034647623698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,51200,5.674400838216146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,10240,1.1299722035725912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,65536,7.178047180175781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,8192,0.4699562708536784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,8192,0.847811190287272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,7168,0.4067690531412761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,16384,1.3023700714111328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,7168,0.7382495880126954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,6144,0.6697727839152019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,6144,0.41837441126505537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,5120,0.32037121454874673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,5120,0.5938559850056966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,4096,0.4720320065816243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,10240,0.8420266469319662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,4096,0.27390079498291015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,8192,0.644977060953776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,3584,0.38373759587605794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,51200,5.593066914876302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,3584,0.2504437287648519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,3072,0.382808526357015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,65536,7.459936014811198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,7168,0.5768320083618164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,2560,0.1933461348215739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,2560,0.294979190826416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,2560,0.21783146858215333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,2048,0.23501760164896646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,6144,0.4949322700500488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,2048,0.13095146814982098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,1536,0.15960639317830402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,2048,0.17174612681070964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,1536,0.10609599749247234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,1024,0.11188693046569824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,1024,0.08348480065663656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,3072,0.21060585975646973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,768,0.08932586510976157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,768,0.07386346658070883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,512,0.06792213122049967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,5120,0.41492586135864257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,512,0.06776426633199056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,256,0.05380906661351522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,512,0.08803733189900717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,256,0.0664576013882955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,128,0.05327893495559692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,128,0.06513919830322265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,128,0.060661331812540686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,4096,0.3302709261576334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,64,0.05283626715342203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,51200,32,0.05296639998753866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,64,0.07084693113962809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,51200,32,0.06523413260777791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,3584,0.2916629473368327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,3072,0.24904534022013344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,65536,1.2855167388916016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,1536,0.1338784058888753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,1024,0.10336746374766033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,768,0.09492373466491699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,65536,2.3157941182454427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,51200,256,0.06539306640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,16384,0.5084106763203938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,16384,0.3306314786275228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,51200,0.966701889038086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,12288,0.38415892918904626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,12288,0.24703572591145834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,10240,0.20642239252726235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,10240,0.344158935546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,51200,1.8351573944091797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,8192,0.27922026316324866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,8192,0.1948970635732015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,7168,0.23867413202921547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,7168,0.14878400166829425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,6144,0.19766079584757487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,6144,0.13037119706471761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,65536,1.723915735880534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,16384,0.40741866429646806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,5120,0.20259626706441244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,5120,0.13048746585845947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,12288,0.31584211985270183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,51200,1.4642645517985025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,4096,0.15329066912333172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,4096,0.09657920201619466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,3584,0.1259392023086548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,3584,0.08716906706492106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,3072,0.10186879634857178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,10240,0.2722421328226725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,3072,0.07676160335540771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,2560,0.08632106781005859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,2560,0.06687680085500082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,8192,0.20684800148010254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,2048,0.07133119901021322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,7168,0.17688746452331544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,2048,0.05763520002365112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,1536,0.055810133616129555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,1536,0.05006080071131388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,6144,0.1566090742746989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,1024,0.04042773246765137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,1024,0.0413269321123759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,5120,0.1322976032892863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,768,0.03351253271102905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,4096,0.10493866602579753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,768,0.0371946652730306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,768,0.03453119993209839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,512,0.02611306707064311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,512,0.03223359982172648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,256,0.018150399128595986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,3584,0.09294613202412924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,256,0.028573866685231524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,3072,0.08136959870656332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,128,0.014115200440088908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,128,0.027332266171773274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,64,0.012462932864824932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,2560,0.07130133310953776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,64,0.027273599306742353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,32,0.013090133666992188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,32,0.026845866441726686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,2048,0.05679253339767456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,1536,0.04767573277155558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,1024,0.03833173513412476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,65536,0.9636575698852539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,65536,1.3205888112386068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,65536,1.6151988983154297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,51200,1.2866111755371095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,16384,0.26644800504048666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,16384,0.4222912152608235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,51200,0.7646240234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,12288,0.28523200352986655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,12288,0.20986347198486327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,512,0.028318933645884198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,128,0.022221867243448892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,256,0.024052266279856363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,10240,0.2493610699971517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,10240,0.17060160636901855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,10240,0.19591253598531086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,7168,0.1741322676340739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,8192,0.19115306536356608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,8192,0.13032853603363037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,7168,0.11689813137054443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,6144,0.14505173365275065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,6144,0.10199466546376545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,5120,0.12191039721171062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,5120,0.08791466554005942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,5120,0.09919466972351074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,4096,0.10005333423614501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,4096,0.07639573415120443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,3584,0.08743999799092611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,3584,0.06850240230560303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,16384,0.30058666865030925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,3072,0.07609919706980386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,51200,0.9666186650594076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,3072,0.06342826684316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,12288,0.22647360165913902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,2560,0.05437333186467489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,8192,0.15416000684102377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,7168,0.14002453486124675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,6144,0.11652373472849528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,2048,0.053922132651011145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,2048,0.04653653303782145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,1536,0.04143146673838298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,1536,0.04053119818369548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,4096,0.0814197301864624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,1024,0.02969493269920349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,1024,0.03286079963048299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,3584,0.07086293697357178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,2560,0.06432213385899863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,768,0.024338134129842124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,768,0.03001813292503357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,512,0.01873813271522522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,512,0.026892799139022826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,3072,0.0611402670542399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,256,0.01328426698843638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,256,0.022476800282796226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,2560,0.05431146621704101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,128,0.010009599725405376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,128,0.02143359978993734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,64,0.008428800106048583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,64,0.02155839999516805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,2048,0.045850666364034016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,32,0.008742400010426839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,32,0.02150826652844747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,1536,0.035558398564656576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,768,0.02378666599591573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,1024,0.03097493251164754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,512,0.02004479964574178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,65536,0.8854741414388021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,65536,1.425814437866211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,16384,0.3524661382039388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,51200,1.0106314977010091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,16384,0.2455754597981771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,51200,0.8181493123372396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,12288,0.17461759249369305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,12288,0.2734112103780111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,12288,0.19494400024414063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,10240,0.15927893320719402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,10240,0.20889172554016114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,8192,0.185261869430542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,8192,0.12402239640553792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,256,0.01634239951769511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,7168,0.14764906565348307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,7168,0.10533973375956218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,8192,0.14840106964111327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,128,0.014144000411033631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,51200,0.6717589060465495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,6144,0.09503253300984701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,5120,0.08120106856028239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,5120,0.10651626586914062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,5120,0.08697173595428467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,4096,0.08647147019704184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,4096,0.06925866603851319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,4096,0.0704650640487671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,3584,0.07612480322519938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,3584,0.06218239863713583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,3584,0.06315840085347493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,3072,0.06636586586634317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,3072,0.056518399715423585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,2560,0.056184534231821695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,2560,0.04999786615371704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,6144,0.12598506609598797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,2048,0.04620586633682251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,2048,0.04302186568578084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,65536,1.0334133148193358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,1536,0.03847786585489909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,1536,0.044123733043670656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,16384,0.26593066851298014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,1024,0.026708267132441205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,1024,0.031056000789006548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,10240,0.1773226737976074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,768,0.021398399273554483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,768,0.027888000011444092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,6144,0.09957333405812582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,512,0.016603733102480568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,7168,0.12147306601206462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,512,0.02498133381207784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,512,0.018422400951385497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,256,0.011749333143234253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,256,0.015727999806404113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,256,0.024426666895548503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,128,0.010453333457310993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,128,0.021989333629608154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,128,0.013432533542315165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,64,0.009674666325251262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,64,0.02258346676826477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,32,0.010449066758155823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,32,0.022711465756098427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,3072,0.05353920062383016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,2560,0.047982935110727945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,65536,0.7130666732788086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,65536,1.051579729715983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,2048,0.04109653234481812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,51200,0.8488970438639323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,1536,0.032687999804814655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,16384,0.26105599403381347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,51200,0.5674879709879558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,1024,0.026921600103378296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,12288,0.197817595799764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,16384,0.20954453150431313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,12288,0.14325013160705566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,10240,0.18198399543762206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,10240,0.12207146485646565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,8192,0.13655786514282225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,8192,0.10126187006632488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,768,0.02211946646372477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,7168,0.11664426326751709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,7168,0.08797333240509034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,6144,0.09993706544240316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,6144,0.07863466739654541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,10240,0.14163734118143717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,51200,0.6956544240315755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,5120,0.08436480363210043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,5120,0.08350186347961426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,4096,0.07112212975819907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,4096,0.06324053208033244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,65536,0.859930674235026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,12288,0.16528213818868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,3584,0.07088426748911539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,3584,0.05268799861272176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,3072,0.052902400493621826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,3072,0.04760533173878988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,16384,0.20850133895874023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,2560,0.045317331949869796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,2560,0.0422165314356486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,8192,0.10278080304463703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,7168,0.09477333227793375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,2048,0.0372927983601888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,2048,0.03632426659266154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,1536,0.02831253409385681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,1536,0.0313536008199056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,6144,0.08055466810862223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,1024,0.02055786649386088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,1024,0.027592533826828004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,5120,0.06758826573689779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,768,0.017112533251444496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,768,0.0253440002600352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,2560,0.03850133419036865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,512,0.01344319979349772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,512,0.02283733288447062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,4096,0.055166931947072353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,256,0.009326933821042379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,256,0.020118399461110433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,3584,0.049388798077901204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,128,0.007523199915885926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,128,0.019884800910949706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,2048,0.03201173345247905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,64,0.00653653343518575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,64,0.01911466717720032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,3072,0.04320000012715657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,32,0.006779733300209046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,32,0.01910933256149292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,1536,0.026983465751012164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,65536,0.6505194981892903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,65536,0.9796789169311524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,51200,0.740823491414388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,51200,0.5564757029215495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,16384,0.2354976018269857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,16384,0.18641173044840495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,1024,0.021614933013916017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,12288,0.1705130736033122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,12288,0.12609492937723796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,256,0.013658666610717773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,10240,0.14513707160949707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,128,0.012865066528320312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,10240,0.10831573009490966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,768,0.019668267170588175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,8192,0.1171562671661377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,8192,0.08928960164388021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,7168,0.10188053448994953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,512,0.01641813317934672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,7168,0.08060906728108724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,6144,0.08820799986521402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,6144,0.07206506729125976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,65536,0.763585090637207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,5120,0.07535466353098551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,5120,0.06311573187510172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,12288,0.13824000358581542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,4096,0.05984213352203369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,16384,0.1906496047973633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,4096,0.05334186553955078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,3584,0.05298346678415934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,3584,0.0482154647509257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,6144,0.07143359978993734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,3072,0.04599786599477132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,51200,0.5871007919311524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,3072,0.043848534425099686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,2560,0.039204267660776775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,2560,0.03941119909286499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,10240,0.12270293235778809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,2048,0.032381866375605264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,2048,0.03372160196304321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,7168,0.08539626598358155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,1536,0.02536639968554179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,1536,0.03007253408432007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,8192,0.09858559767405192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,1024,0.018313600619633993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,1024,0.025142399470011394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,3072,0.038973867893218994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,768,0.014990933736165366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,768,0.023348265886306764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,4096,0.051992531617482504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,512,0.011666133006413778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,512,0.02223680019378662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,1536,0.02350826660792033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,256,0.008820266524950663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,256,0.020520534118016562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,5120,0.061717331409454346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,128,0.007764266431331634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,2560,0.03571306864420573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,128,0.01877760092417399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,64,0.007358933488527934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,64,0.019067732493082683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,3584,0.04480533202489217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,32,0.007630933324495952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,32,0.01914560000101725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,65536,0.5655082702636719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,65536,0.7574783960978191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,51200,0.6335786819458008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,2048,0.02916266719500224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,51200,0.4630122820536296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,16384,0.19109439849853516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,16384,0.15842026074727375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,128,0.009925333658854167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,12288,0.14122452735900878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,256,0.010939733187357584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,12288,0.11278613408406575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,10240,0.11842559973398845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,512,0.013173333803812661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,10240,0.09693013032277426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,1024,0.020242132743199668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,8192,0.09569706916809081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,8192,0.08065600395202636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,7168,0.085370667775472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,7168,0.07296960353851319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,768,0.01617173353830973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,6144,0.0732362667719523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,6144,0.06515520016352336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,65536,0.7497706731160482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,5120,0.06391786734263102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,5120,0.060236799716949466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,16384,0.18616533279418945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,4096,0.05021653175354004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,4096,0.048995200792948404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,51200,0.5935359954833984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,3584,0.045208533604939774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,3584,0.04614080190658569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,12288,0.14380586942036946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,3072,0.03915839989980062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,3072,0.04007999897003174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,6144,0.06724053223927816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,10240,0.11639359792073567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,2560,0.03322133421897888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,2560,0.036050132910410565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,2048,0.02810773253440857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,2048,0.032601600885391234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,7168,0.07522239685058593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,1536,0.021937066316604616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,1536,0.028264532486597698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,8192,0.08386773268381754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,1024,0.016337066888809204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,1024,0.024759467442830405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,3072,0.03670719861984253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,768,0.013849600156148275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,768,0.02243946592013041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,5120,0.05852160056432089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,512,0.011156266927719117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,512,0.020733867088953653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,4096,0.0469813346862793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,256,0.007957333326339721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,256,0.018784000476201376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,3584,0.04016213417053223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,128,0.0066101332505544026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,128,0.018132267395655315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,1536,0.021924267212549843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,64,0.005789866546789805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,64,0.01840959986050924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,32,0.006043733159701029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,32,0.018372267484664917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,2560,0.03174293239911397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,65536,0.651531728108724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,65536,0.5203989346822102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,2048,0.026925865809122724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,1024,0.018397865692774455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,51200,0.5066666603088379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,51200,0.4160650571187337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,16384,0.16495787302652995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,51200,0.43863786061604815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,16384,0.13763200441996257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,12288,0.12421440283457438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,16384,0.1409653345743815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,10240,0.10446613629659016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,12288,0.10542079607645671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,10240,0.09791786670684814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,8192,0.08625813325246176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,8192,0.07655573685963948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,10240,0.08920853137969971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,256,0.010940800110499065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,128,0.009910399715105694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,512,0.012894933422406515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,7168,0.06688533624013265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,7168,0.06776746908823648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,6144,0.06331946849822997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,6144,0.060702931880950925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,6144,0.05422186851501465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,5120,0.05454080104827881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,5120,0.05299199819564819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,5120,0.04787840048472087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,4096,0.043747198581695554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,4096,0.045441067218780516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,768,0.0152319997549057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,3584,0.039637335141499835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,3584,0.041155199209849044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,3072,0.03440746863683065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,3072,0.03714453379313151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,3072,0.030979200204213457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,2560,0.029397332668304445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,2560,0.033555201689402264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,2560,0.028091732660929365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,7168,0.07341226736704508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,2048,0.024388267596562704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,2048,0.02405440012613932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,2048,0.02967146635055542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,1536,0.020051199197769164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,1536,0.02686506708463033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,1536,0.019466666380564372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,1024,0.014803199966748556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,1024,0.023151999711990355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,1024,0.016591999928156534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,768,0.012583466370900473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,768,0.02114880084991455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,768,0.013724799950917563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,512,0.009754666686058044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,512,0.019642666975657145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,256,0.007299200197060903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,256,0.018498132626215615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,256,0.010071466366449993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,128,0.006116266548633576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,128,0.017691733439763387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,64,0.00537066658337911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,64,0.017859200636545815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,32,0.005636266867319743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,32,0.01763946612675985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,65536,0.5168181419372558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,12288,0.10528319676717121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,65536,0.4811402638753255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,51200,0.403766409556071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,65536,0.5551402409871419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,51200,0.39219306310017904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,128,0.009057066837946574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,16384,0.13143359820048015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,16384,0.11660160223642986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,51200,0.40917228062947586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,12288,0.10008426507314046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,12288,0.10043839613596599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,10240,0.09000426928202311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,10240,0.08733226458231608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,512,0.011292800307273865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,8192,0.06667093435923258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,8192,0.0658570647239685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,8192,0.07525226275126139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,7168,0.06044586499532064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,7168,0.06176640192667643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,65536,0.5410442352294922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,6144,0.050800001621246337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,6144,0.053257600466410315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,16384,0.13063466548919678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,4096,0.039238401254018146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,12288,0.10577066739400227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,5120,0.04314879973729451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,5120,0.0464469313621521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,4096,0.035266133149464925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,4096,0.039662933349609374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,3584,0.030892799297968548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,3584,0.0353983998298645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,10240,0.09146239757537841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,3072,0.02747093240420024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,3072,0.03303359945615132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,3584,0.03554240067799886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,2560,0.02344320019086202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,2560,0.029639466603597002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,8192,0.06173333326975504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,2048,0.01909439961115519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,2048,0.026870399713516235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,7168,0.05539199908574423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,1536,0.015938133001327515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,1536,0.024630399545033772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,6144,0.049293867746988934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,1024,0.012168533603350322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,1024,0.020812799533208214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,1024,0.014257066448529563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,768,0.010101333260536194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,768,0.020269866784413657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,5120,0.041162665685017905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,512,0.008083199958006541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,512,0.01880106727282206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,4096,0.03516053358713786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,3584,0.029496532678604127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,256,0.006232533355553945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,256,0.017574399709701538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,3072,0.027419734001159667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,128,0.005339733262856802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,128,0.017090133825937905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,2560,0.024243199825286867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,64,0.004640000065167745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,64,0.017078399658203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,32,0.004891733328501383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,2048,0.020228266716003418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,1536,0.01699519952138265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,65536,0.44733225504557295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,768,0.011805867155392964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,65536,0.423583984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,51200,0.34883305231730144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,51200,0.33409919738769533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,32,0.016956800222396852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,16384,0.11734506289164226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,16384,0.1085546652475993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,512,0.010128000378608703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,12288,0.08754560152689615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,12288,0.08448959986368815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,256,0.008946133653322856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,10240,0.07359786828358969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,10240,0.07254292964935302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,8192,0.05857813358306885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,128,0.008211199939250947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,8192,0.06317866643269857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,7168,0.05234346787134806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,7168,0.05644586483637491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,6144,0.0452672004699707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,6144,0.05056213140487671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,12288,0.09111680189768473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,5120,0.03860053221384684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,5120,0.04405226707458496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,65536,0.5155530611673991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,16384,0.12315306663513184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,4096,0.03218666712443034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,4096,0.03805653254191081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,6144,0.04517120122909546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,3584,0.02855786681175232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,3584,0.034704001744588216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,3072,0.024474666515986124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,3072,0.03201813300450643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,10240,0.08585706551869711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,2560,0.021156267325083414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,2560,0.029893332719802858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,7168,0.0534442663192749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,2048,0.017998933792114258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,51200,0.3907093365987142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,2048,0.026870399713516235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,1536,0.014868266383806863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,1536,0.024154667059580484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,8192,0.0582261323928833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,1024,0.011507200201352437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,1024,0.021614933013916017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,3072,0.025588266054789227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,768,0.010237866640090942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,768,0.020337067047754922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,1536,0.016708266735076905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,512,0.007997866471608479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,512,0.01912213365236918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,4096,0.033192533254623416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,256,0.00647680014371872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,2560,0.023128533363342287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,5120,0.03902080059051514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,128,0.005629866818586985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,128,0.0172160009543101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,3584,0.02924906611442566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,64,0.005117866893609365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,64,0.017288533846537273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,32,0.005352533360322317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,1024,0.013853866855303446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,32,0.01762346625328064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,2048,0.020100265741348267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,65536,0.38234453201293944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,65536,0.40351146062215165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,256,0.01802133321762085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,51200,0.3167114575703939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,16384,0.09810346762339274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,51200,0.3085653305053711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,16384,0.1040981372197469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,12288,0.07461439768473307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,12288,0.0809781312942505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,256,0.008719999591509502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,10240,0.06351999839146932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,512,0.010149332880973815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,10240,0.07110186417897543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,8192,0.051292800903320314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,128,0.008007466793060303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,8192,0.058821332454681394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,7168,0.045390931765238444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,768,0.011673600474993388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,7168,0.052907733122507725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,6144,0.03964586655298869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,6144,0.04710293213526408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,12288,0.09030186335245768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,5120,0.03383040030797323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,5120,0.04079573154449463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,65536,0.4879317283630371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,16384,0.11869653065999348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,4096,0.027831466992696126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,4096,0.03519573211669922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,51200,0.39828694661458336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,3584,0.02405760089556376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,3584,0.032568534215291337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,3072,0.021202133099238078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,3072,0.029624533653259278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,6144,0.043628799915313723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,2560,0.01831679940223694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,2560,0.027005867163340254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,10240,0.08122986952463786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,2048,0.015473066767056783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,2048,0.024834134181340537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,7168,0.0512223998705546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,1536,0.012757333119710288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,1536,0.023014400402704874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,8192,0.05681386788686117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,1024,0.009883733590443929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,1024,0.020028799772262573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,4096,0.030899200836817426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,768,0.008366933465003968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,768,0.018994132677714028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,5120,0.03699306646982829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,512,0.006764799853165944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,512,0.01807360053062439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,3072,0.0234442671140035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,256,0.005458133419354757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,256,0.016670932372411094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,1536,0.015242666999499003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,128,0.004647466540336609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,128,0.016451199849446617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,3584,0.02701653242111206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,64,0.004265599946180979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,64,0.016463999946912132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,32,0.004445866743723551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,2560,0.02186453342437744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,32,0.016551466782887776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,65536,0.33941332499186194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,65536,0.3589248021443685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,2048,0.018142932653427125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,65536,0.3266805330912272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,51200,0.26445013682047525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,16384,0.08566293716430665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,51200,0.2843370755513509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,16384,0.0934058666229248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,51200,0.25981119473775227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,12288,0.06500053405761719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,12288,0.07470400333404541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,12288,0.07348586718241373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,10240,0.05541226863861084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,10240,0.06452586650848388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,10240,0.06578559875488281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,8192,0.04413546721140544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,8192,0.055137066046396885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,8192,0.046077867348988846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,7168,0.039468801021575926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,1024,0.012442666292190551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,7168,0.04917866786321004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,7168,0.04453759988149007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,6144,0.034186665217081705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,6144,0.03834026654561361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,6144,0.04329280058542888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,5120,0.02912213404973348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,4096,0.023821866512298583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,5120,0.031194667021433514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,5120,0.03694933255513509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,3584,0.021102933088938396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,4096,0.027059199412663777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,4096,0.03258986671765645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,3072,0.018564265966415406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,3072,0.02827306588490804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,3072,0.021597866217295328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,2560,0.0162090669075648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,2560,0.02577706575393677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,512,0.009388800462086995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,2048,0.013797332843144735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,768,0.01069119970003764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,2048,0.02363626758257548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,1536,0.01156160036722819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,1536,0.021001599232355752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,3584,0.029677865902582805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,1024,0.008803199728329976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,1024,0.019099734226862588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,256,0.00827519992987315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,768,0.0076447998483975725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,768,0.01886506676673889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,768,0.009237333138783773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,512,0.00626986672480901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,512,0.01747093399365743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,512,0.008135466774304708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,256,0.005099733173847198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,256,0.016570666432380678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,128,0.007464533547560374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,16384,0.08402559757232667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,128,0.004469333092371622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,128,0.006764799853165944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,128,0.016872533162434897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,64,0.004264533519744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,64,0.01635840038458506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,32,0.0044159998496373495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,2048,0.015990400314331056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,32,0.016104533274968465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,65536,0.26962668100992837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,65536,0.3326730728149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,51200,0.21037866274515787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,51200,0.2737802823384603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,2560,0.01888426740964254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,256,0.007387733459472657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,16384,0.09401600360870362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,16384,0.09897600015004476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,12288,0.07368000348409018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,16384,0.0882368008295695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,12288,0.07816212972005208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,10240,0.060933331648508705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,10240,0.06759359836578369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,10240,0.055829334259033206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,8192,0.045813333988189694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,8192,0.051520001888275144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,7168,0.042786133289337155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,7168,0.046020265420277914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,8192,0.046248535315195724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,6144,0.03699306646982829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,6144,0.041048534711201984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,6144,0.035104000568389894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,5120,0.03183253407478333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,5120,0.03602026700973511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,65536,0.3365450541178385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,4096,0.020334933201471964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,4096,0.030906667311986286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,7168,0.04053973356882731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,1536,0.012762666742006937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,3584,0.019245866934458414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,3584,0.02834986646970113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,3072,0.017937066157658894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,3072,0.026529065767923993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,3072,0.018940800428390504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,2560,0.014523733655611673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,2560,0.02477653423945109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,51200,0.2627583980560303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,2048,0.012131200234095255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,2048,0.021670399109522502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,2048,0.01427839994430542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,1536,0.009938133756319682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,1536,0.020347734292348228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,12288,0.06567146778106689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,1024,0.007277866701285045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,5120,0.02974933385848999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,1024,0.01872106591860453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,768,0.006428800026575724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,768,0.017871999740600587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,1024,0.010811733206113179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,512,0.005456000069777171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,3584,0.02267306645711263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,512,0.007449600100517273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,256,0.004474666714668274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,256,0.015890133380889893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,256,0.006742399930953979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,128,0.003973333289225897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,128,0.01572266618410746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,4096,0.025174399216969807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,64,0.003717333326737086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,64,0.015640532970428465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,32,0.003913599997758865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,32,0.015586133797963461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,3584,0.02177919944127401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,65536,0.2048405329386393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,65536,0.30160001118977864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,2560,0.016938666502634682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,51200,0.1591946601867676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,512,0.01729066570599874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,51200,0.23941119511922201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,16384,0.058500266075134276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,16384,0.07983573277791342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,16384,0.07715840339660644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,1536,0.0113237331310908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,12288,0.04198933442433675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,10240,0.03552853266398112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,12288,0.06426986853281656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,10240,0.054554665088653566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,8192,0.029011199871699016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,8192,0.0446730653444926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,8192,0.03945813179016113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,7168,0.025814400116602583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,7168,0.03940693140029907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,1024,0.009751466910044353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,6144,0.02228053410847982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,768,0.008393599589665731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,6144,0.03559786478678385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,5120,0.01885653336842855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,5120,0.03174506624539693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,128,0.006286933521429698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,4096,0.015843199690183003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,4096,0.027898667256037395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,65536,0.29937171936035156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,3584,0.014602667093276978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,3584,0.02643306652704875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,51200,0.23944212595621744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,12288,0.05702613194783529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,3072,0.012964266538619994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,3072,0.024973867336908977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,3072,0.016396799683570863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,10240,0.05141439835230509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,2560,0.01151146690050761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,2560,0.02302079995473226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,2048,0.012489599982897441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,2048,0.02108373244603475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,7168,0.034295467535654704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,1536,0.009898666540781658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,1536,0.02183786630630493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,6144,0.03030719955762227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,1024,0.0075573335091273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,5120,0.026078933477401735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,1024,0.01858453353246053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,768,0.006460799773534138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,768,0.01820053259531657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,4096,0.021314134200414024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,512,0.005240533252557119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,512,0.016643200318018594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,3584,0.01795519987742106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,256,0.004311466713746389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,256,0.01590826710065206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,256,0.006497066716353099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,128,0.0037962667644023894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,128,0.01601920028527578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,64,0.003638399889071783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,64,0.015596800049146018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,2560,0.014468266566594442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,32,0.01599360009034475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,2048,0.012105600039164225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,65536,0.1397216002146403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,1536,0.010588799913724262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,65536,0.266046937306722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,51200,0.1102783997853597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,51200,0.22285226186116538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,16384,0.040856532255808514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,16384,0.08738133112589518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,1024,0.009040000041325887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,12288,0.053082664807637535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,128,0.005936000247796377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,12288,0.06716799736022949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,768,0.00812799980243047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,10240,0.04252266486485799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,32,0.0036917333801587426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,10240,0.051319468021392825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,8192,0.035656531651814774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,8192,0.04049280087153117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,7168,0.030777599414189654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,7168,0.03654719988505046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,512,0.007049599786599477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,6144,0.025917865832646686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,6144,0.0335968017578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,65536,0.2691338539123535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,12288,0.051013334592183435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,5120,0.02980479995409648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,16384,0.06296106576919555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,51200,0.21126079559326172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,4096,0.01455466647942861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,4096,0.026638933022816974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,3584,0.012089600165685017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,10240,0.042558932304382326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,3584,0.024873600403467814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,3072,0.011771733562151592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,3072,0.022825600703557332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,8192,0.03038613398869832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,2560,0.009471999605496724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,2560,0.021434666713078816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,7168,0.027561599016189577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,2048,0.008054399987061818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,5120,0.018305067221323648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,2048,0.01993066668510437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,1536,0.006883200009663899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,1536,0.018989866971969603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,6144,0.024366933107376098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,1024,0.005646933118502299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,1024,0.017620267470677693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,3072,0.014362667004267374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,768,0.0049002667268117275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,768,0.016999467213948568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,5120,0.021169066429138184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,512,0.004385066529115042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,4096,0.017595734198888144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,512,0.01653333306312561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,256,0.0038218667109807336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,3584,0.016193067034085594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,256,0.015507200360298156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,128,0.003455999990304311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,128,0.01585919956366221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,1536,0.00916266640027364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,2560,0.012402133146921793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,64,0.003302400062481562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,32,0.0034527999659379324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,64,0.015294933319091797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,32,0.015296000242233276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,65536,0.11463039716084797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,65536,0.25014185905456543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,51200,0.08904213110605876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,2048,0.010737066467603047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,51200,0.19976746241251628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,1024,0.00806826651096344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,16384,0.03394986788431804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,16384,0.07237226963043213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,256,0.005704533557097117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,12288,0.028217599789301558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,128,0.005312000215053558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,12288,0.05471893151601156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,10240,0.026768000920613606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,768,0.007186133166154225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,10240,0.04527146816253662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,8192,0.01915839910507202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,512,0.006321066617965698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,8192,0.0370741327603658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,7168,0.01736853321393331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,7168,0.03400426705678304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,6144,0.015095466375350952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,6144,0.03100373347600301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,65536,0.2351466655731201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,5120,0.013487999637921652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,5120,0.030763733386993408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,51200,0.18382933934529622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,16384,0.057144534587860105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,4096,0.011014399925867717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,4096,0.02754240036010742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,3584,0.009980799754460652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,3584,0.02551146745681763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,12288,0.0420693318049113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,3072,0.008905599514643352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,3072,0.023798400163650514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,6144,0.02236479918162028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,2560,0.011335466305414836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,2560,0.02213546633720398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,10240,0.03652053276697795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,2048,0.009619200229644775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,2048,0.020167466004689535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,7168,0.02566933234532674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,1536,0.008036266764005024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,1536,0.018879999717076622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,8192,0.028807467222213744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,1024,0.006137600044409434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,1024,0.017489065726598106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,5120,0.01949333349863688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,768,0.005319466690222422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,768,0.01720213294029236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,4096,0.016238933801651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,512,0.0045514668027559916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,512,0.016452266772588094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,3584,0.014993066589037577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,256,0.0038453333079814913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,256,0.01567466656366984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,3072,0.013294933239618936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,128,0.0034272000193595886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,128,0.015080533425013223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,1536,0.008929066856702169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,64,0.00323840007185936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,2560,0.011914666493733723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,32,0.0031989333530267083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,32,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,65536,0.07823999722798666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,65536,0.23733013470967612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,2048,0.010455466310183207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,1024,0.007900799810886382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,51200,0.0637450655301412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,51200,0.19252479871114095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,16384,0.027360000212987262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,16384,0.06874879995981852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,64,0.015341867009798685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,12288,0.02153600056966146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,12288,0.05075733264287313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,768,0.006949333349863689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,10240,0.01981546680132548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,10240,0.042448000113169355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,512,0.006291200220584869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,8192,0.016518400112787882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,8192,0.0352725346883138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,256,0.005736533304055532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,7168,0.01534293293952942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,7168,0.03234773278236389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,128,0.005312000215053558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,6144,0.013457066814104714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,6144,0.02947733402252197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,65536,0.21185919443766274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,5120,0.011969066659609477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,5120,0.02677653431892395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,12288,0.04041920105616252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,51200,0.18067092895507814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,4096,0.010086400310198466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,16384,0.05236266851425171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,4096,0.024375466505686443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,3584,0.008935466408729553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,3584,0.022845866282780965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,3072,0.008274133503437042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,3072,0.021543467044830324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,10240,0.035717332363128663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,2560,0.007463466624418895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,2560,0.02178879976272583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,8192,0.027204267183939618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,6144,0.021001599232355752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,2048,0.00619946668545405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,2048,0.01932586630185445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,7168,0.024962133169174193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,1536,0.005782400071620941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,1536,0.01858453353246053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,3072,0.012859732906023661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,1024,0.0046528001626332605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,1024,0.01730133295059204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,768,0.004214400053024292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,768,0.016682666540145875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,5120,0.018641066551208497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,512,0.0037994667887687682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,4096,0.015427199999491372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,512,0.01634773313999176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,256,0.0033919999996821085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,256,0.015433599551518759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,3584,0.014029866456985474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,128,0.0031818665564060213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,2560,0.011612799763679505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,128,0.015254400173823037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,64,0.0030037333567937215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,64,0.015253333250681558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,32,0.003147733211517334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,32,0.015233066678047181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,2048,0.00986346701780955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,1536,0.008597333232561748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,65536,0.057526401678721105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,51200,0.04649279912312825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,65536,0.2241578737894694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,51200,0.17966507275899252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,16384,0.016105600198109946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,51200,0.13665493329366046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,16384,0.06243093411127726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,12288,0.01529813309510549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,12288,0.04697493314743042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,768,0.006859733164310456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,10240,0.013736533125241599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,10240,0.03958293199539185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,65536,0.17035306294759114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,8192,0.011879466970761617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,1024,0.007639466722806294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,8192,0.0334826668103536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,16384,0.045415465037027994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,7168,0.01125440001487732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,7168,0.031167999903361006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,6144,0.011866666873296102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,6144,0.028008532524108887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,12288,0.035469865798950194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,5120,0.010185600320498148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,512,0.006133333345254262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,5120,0.025518933931986492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,4096,0.008689066767692566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,256,0.005691733459631602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,4096,0.024405332406361897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,3584,0.007949866851170858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,128,0.0052138666311899815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,3584,0.022789333264033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,3072,0.007227733234564463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,3072,0.02126400073369344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,10240,0.030065067609151203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,2560,0.006773333251476288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,2560,0.020253866910934448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,8192,0.0238154669602712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,2048,0.005906133353710175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,2048,0.018629332383473717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,6144,0.018927999337514243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,7168,0.021422932545344033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,1536,0.005206400156021118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,1536,0.01787733236948649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,1024,0.004500266909599304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,1024,0.016926934321721397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,5120,0.016477866967519125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,768,0.004042666653792063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,768,0.016863999764124553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,4096,0.01404159963130951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,512,0.0036746665835380556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,3584,0.01299626628557841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,512,0.01599253316720327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,512,0.005883733431498209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,256,0.003306666761636734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,256,0.0053045332431793215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,256,0.015069866180419922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,128,0.0030645333230495454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,3072,0.011884799599647522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,64,0.002962133288383484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,128,0.01530239979426066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,32,0.0029685333371162414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,64,0.01479680041472117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,32,0.014685866236686707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,65536,0.047622398535410566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,65536,0.217576535542806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,51200,0.03933759927749634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,51200,0.17277973492940266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,2560,0.010852266351381938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,16384,0.010681600371996561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,2048,0.009116799632708231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,16384,0.058234667778015135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,12288,0.009237333138783773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,12288,0.042022399107615155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,1536,0.008365866541862488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,10240,0.010345600048700969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,10240,0.03503679831822713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,1024,0.007253333429495494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,768,0.006321066617965698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,8192,0.00899839997291565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,8192,0.0302293340365092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,128,0.004846933484077454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,7168,0.008648533622423809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,7168,0.0310805340607961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,65536,0.1692992051442464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,51200,0.13287466367085773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,6144,0.008117333551247915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,6144,0.027747199932734175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,5120,0.00795839975277583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,16384,0.04524053335189819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,12288,0.034397868315378825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,5120,0.025405865907669068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,4096,0.007269333302974701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,4096,0.023044266303380332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,3584,0.007699200014273326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,3584,0.021784534056981407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,3072,0.007001600166161854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,10240,0.028602667649586994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,3072,0.02076479991277059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,2560,0.006503466765085857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,8192,0.02344533403714498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,7168,0.02121493419011434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,2560,0.019333332777023315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,2048,0.005568000177542368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,2048,0.018807466824849448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,1536,0.00489279975493749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,1536,0.018257067600886027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,6144,0.01875306765238444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,1024,0.00413973331451416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,1024,0.01728746692339579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,5120,0.016195199886957803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,768,0.0037941334148248037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,768,0.016218666235605875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,4096,0.013945600390434265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,3584,0.012851199507713318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,512,0.003533866753180822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,512,0.015770666797955833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,256,0.003139200061559677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,256,0.015135999520619711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,3072,0.011639466881752015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,128,0.003014400104681651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,128,0.01504746675491333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,2560,0.01056106686592102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,64,0.0027818667391935987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,64,0.01474240024884542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,32,0.002812800059715907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,32,0.014613333344459533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,2048,0.008949333429336548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,65536,0.043093331654866535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,51200,0.03509653409322103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,65536,0.2154933293660482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,16384,0.009501866499582927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,51200,0.1710101286570231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,16384,0.05586666663487753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,12288,0.007991466422875721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,10240,0.008983467022577922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,12288,0.03951786756515503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,1536,0.008272000153859456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,8192,0.008088533580303193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,10240,0.033207466204961136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,8192,0.028939733902613323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,6144,0.007124266525109608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,7168,0.02815999984741211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,5120,0.006679466863473256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,6144,0.026392533381779985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,4096,0.006322133541107178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,5120,0.025433599948883057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,3584,0.006674133241176605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,4096,0.02401919960975647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,3072,0.006281599899133046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,3584,0.022485333681106567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,2560,0.00621013343334198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,3072,0.020950400829315187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,2048,0.005427200098832448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,2560,0.019834667444229126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,1536,0.004755199948946635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,2048,0.01844053268432617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,1024,0.0040778666734695435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,1536,0.01764906644821167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,768,0.0038624001046021783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,1024,0.01700053413709005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,512,0.003443199892838796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,768,0.01630826691786448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,256,0.003188266605138779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,512,0.015618133544921874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,768,0.0063498665889104204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,128,0.002919466545184453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,256,0.015316266814867655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,64,0.002829866607983907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,128,0.014664533734321594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,32,0.0028309332827727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,64,0.014786133170127868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,32,0.01458560029665629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,65536,0.040763731797536215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,51200,0.03342613379160563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,16384,0.008693333466847737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,16384,0.053725866476694736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,51200,0.16987412770589191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,12288,0.00814933329820633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,7168,0.007716266810894013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,10240,0.007503999769687653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,10240,0.03421440124511719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,512,0.005702400207519531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,8192,0.00687360018491745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,7168,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,8192,0.029217066367467244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,7168,0.028163200616836546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,6144,0.006155733267466227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,5120,0.006422399977842967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,6144,0.026474666595458985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,4096,0.006040533383687338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,5120,0.025361067056655882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,256,0.005275733272234599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,3584,0.006489600241184235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,1024,0.007076266904671986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,4096,0.022986666361490885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,3072,0.006166400015354156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,3584,0.021568000316619873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,2560,0.00628053347269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,3072,0.020819199085235596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,2048,0.0053845331072807315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,2560,0.019526400168736777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,1536,0.004794666667779287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,2048,0.01839253306388855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,1024,0.0040501333773136135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,768,0.003751466671625773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,1024,0.01709653337796529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,512,0.003435733417669932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,768,0.016107733050982155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,256,0.0030410667260487873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,128,0.004923733572165171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,512,0.015508266290028891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,128,0.002918400118748347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,256,0.01502826710542043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,64,0.0027295999228954316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,32,0.0026986666023731233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,128,0.015076266725858054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,64,0.01474453310171763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,32,0.014579199751218162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,65536,0.2135039965311686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,12288,0.038943998018900555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,1536,0.018231467405954997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,16384,0.7789503733317058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,16384,1.5810325622558594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,12288,0.6011135737101237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,12288,1.1394176483154297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,10240,0.5186271985371907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,51200,2.519453938802083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,10240,0.9174005508422851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,8192,0.41092586517333984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,8192,0.7491466522216796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,7168,0.3759050687154134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,7168,0.6845397313435873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,51200,5.033079528808594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,6144,0.5934282938639324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,16384,1.7374399820963542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,6144,0.32073065439860027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,5120,0.46600745519002273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,12288,1.2498762766520182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,5120,0.27099199295043946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,4096,0.21892693837483726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,4096,0.3740415891011556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,10240,1.0547957102457681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,3584,0.20357012748718262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,3584,0.34325227737426756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,4096,0.44251413345336915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,3072,0.29710613886515297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,3072,0.17370346387227376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,8192,0.8936597188313803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,2560,0.24026239713033043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,2560,0.1519210656483968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,2048,0.19593599637349446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,2048,0.13320426940917968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,7168,0.7813802719116211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,1536,0.15748052597045897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,1536,0.10976533095041911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,6144,0.6644309361775715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,1024,0.10910507043202718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,1024,0.07874773343404134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,5120,0.5549546559651692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,768,0.08875199953715006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,768,0.06889493465423584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,51200,6.979356892903645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,3584,0.38595625559488933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,512,0.0690399964650472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,512,0.058500266075134276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,256,0.04484800100326538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,256,0.04977173407872518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,3072,0.3341504096984863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,128,0.04150293270746867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,2560,0.2780618667602539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,128,0.045196799437205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,64,0.03224853277206421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,64,0.04540266593297322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,65536,32,0.032323199510574344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,65536,32,0.04613120158513387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,2048,0.2245375951131185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,1536,0.16597332954406738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,1024,0.11440959771474202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,768,0.0994101365407308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,512,0.07717013359069824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,256,0.06548800071080527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,65536,128,0.06090346574783325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,16384,1.215979766845703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,65536,2.9651646931966145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,16384,0.6618293126424153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,51200,2.1900950113932294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,12288,0.5110581398010254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,12288,0.9467925389607748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,65536,5.733099873860677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,51200,4.264921569824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,10240,0.7814133326212566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,10240,0.44191681543986006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,8192,0.37092908223470056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,8192,0.6443456013997395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,8192,0.7069525400797526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,7168,0.5470336278279622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,7168,0.31002880732218424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,6144,0.27127040227254234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,6144,0.4688447952270508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,5120,0.3940266609191895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,16384,1.3438475290934244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,5120,0.23292694091796876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,4096,0.3129664103190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,12288,0.9807392120361328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,4096,0.193234125773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,10240,0.8517845153808594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,3584,0.2774325370788574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,3584,0.17410987218221027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,3072,0.24252692858378092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,3072,0.14649279912312824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,7168,0.6176565170288086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,3072,0.26220906575520836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,2560,0.20471679369608559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,2560,0.12725120385487873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,2048,0.16616640090942383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,2048,0.11495786507924396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,6144,0.5261770566304524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,1536,0.12413866519927978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,65536,7.082323201497395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,1536,0.09263359705607097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,1024,0.08849493662516275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,1024,0.06912639935811361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,51200,5.101436869303385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,5120,0.44150508244832354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,768,0.07532587051391601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,768,0.06095679998397827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,512,0.053191467126210534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,512,0.0514357328414917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,4096,0.3515893300374349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,256,0.037478399276733396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,256,0.04351253509521484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,3584,0.30729707082112634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,128,0.03251840074857076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,128,0.03959893385569255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,2560,0.22070825894673668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,64,0.029899734258651733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,64,0.03984853426615397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,51200,32,0.02656960090001424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,51200,32,0.040705064932505294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,2048,0.17714452743530273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,1536,0.13060692946116131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,1024,0.09018452962239584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,768,0.08012266953786215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,512,0.06280533472696939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,65536,0.8763157526652018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,256,0.053192532062530516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,65536,1.5587701161702474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,16384,0.21369493802388512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,16384,0.3925386746724447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,51200,0.6653770446777344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,51200,128,0.04986879825592041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,51200,1.1403840382893882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,12288,0.2642634709676107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,12288,0.17898133595784504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,10240,0.22467519442240397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,10240,0.17640746434529622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,8192,0.18250239690144857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,8192,0.11902720133463543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,7168,0.16029653549194336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,7168,0.106494935353597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,6144,0.1376853307088216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,6144,0.09268799622853598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,16384,0.4177258809407552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,12288,0.31247361501057946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,10240,0.27158079147338865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,5120,0.07965119679768881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,4096,0.09260586897532144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,51200,1.421219253540039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,8192,0.2286293347676595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,4096,0.06766506830851236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,3584,0.06249493360519409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,3584,0.0821664015452067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,65536,1.9634101867675782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,7168,0.20446720123291015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,3072,0.07118399937947592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,2560,0.06030933459599813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,3072,0.06346346537272135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,2560,0.054890668392181395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,6144,0.1742538611094157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,2048,0.04989120165506999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,2048,0.04396799802780151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,5120,0.11819199721018474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,1536,0.04079039891560872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,1536,0.038427734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,5120,0.14618879954020184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,1024,0.02762773235638936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,1024,0.03118293285369873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,3584,0.10226453145345052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,4096,0.11696746349334716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,768,0.022872533400853476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,768,0.028629332780838013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,768,0.029174399375915528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,3072,0.08693866729736328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,512,0.025725867350896197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,512,0.023668267329533896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,256,0.01245013376077016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,256,0.02252266605695089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,128,0.009525332848230999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,128,0.020858667294184365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,128,0.01869973341623942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,64,0.007975466549396515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,2560,0.07334293524424235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,32,0.008429867029190064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,64,0.021128533283869426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,2048,0.05934613148371378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,16384,32,0.02072319984436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,1536,0.04482346773147583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,16384,512,0.017617066701253258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,65536,0.7580277125040691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,51200,0.9938421249389648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,51200,0.6047957102457683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,65536,1.300502395629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,16384,0.3168896039326986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,16384,0.22221973737080894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,12288,0.16117119789123535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,12288,0.2929343859354655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,10240,0.2006762663523356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,10240,0.15042880376180012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,1024,0.03214613397916158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,8192,0.16329280535380047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,8192,0.11191999912261963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,7168,0.14321386019388835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,7168,0.10162453651428223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,16384,256,0.020321067174275717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,6144,0.12463573614756267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,6144,0.09079360167185466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,6144,0.13691627184549968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,5120,0.10634026527404786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,5120,0.07873706817626953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,65536,1.4625439961751303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,16384,0.3376192092895508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,4096,0.09848000208536783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,12288,0.2526474634806315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,3584,0.077019731203715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,51200,1.0888778686523437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,10240,0.22035306294759116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,3584,0.06091519991556803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,3072,0.06804693539937337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,3584,0.08565973440806071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,3072,0.054162132740020755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,2560,0.0604479988416036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,2560,0.04642133315404256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,8192,0.18517972628275553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,2048,0.04700800180435181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,2048,0.04044373432795207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,1536,0.037598931789398195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,1536,0.03476586739222209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,7168,0.16542506217956543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,1024,0.024965333938598632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,4096,0.06667946974436442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,1024,0.02802773316701253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,768,0.020014933745066323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,1024,0.027091199159622194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,768,0.025678932666778564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,512,0.015000533064206442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,512,0.02339093287785848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,5120,0.11869440078735352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,256,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,256,0.01984213391939799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,4096,0.09522133668263753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,128,0.008090666433175405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,128,0.02015786568323771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,3072,0.0704255978266398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,64,0.007099733253320058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,64,0.02217493255933126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,2560,0.0602837324142456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,12288,32,0.009107200304667155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,2048,0.04892266591389974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,12288,32,0.022052266200383506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,1536,0.03753386735916138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,768,0.022503467400868733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,512,0.01774719953536987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,65536,0.5921930948893229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,65536,0.9555445353190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,16384,0.2304447968800863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,51200,0.4755711873372396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,256,0.014822399616241455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,16384,0.17555732727050782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,12288,0.12693119843800862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,51200,0.7324277242024739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,10240,0.1494474728902181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,12288,128,0.012718933820724487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,10240,0.10210346380869548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,12288,0.17140053113301595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,8192,0.11746666431427003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,8192,0.08406293392181396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,7168,0.10216000080108642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,7168,0.07546986738840739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,6144,0.09097386995951334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,6144,0.06883626778920492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,12288,0.24020160039265953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,5120,0.07445973555246989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,5120,0.059078399340311685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,16384,0.3213386535644531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,65536,1.2905994415283204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,4096,0.05977386633555094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,4096,0.05802666743596395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,3584,0.054605865478515626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,3584,0.04762453238169352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,10240,0.20883092880249024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,51200,0.9876874923706055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,3072,0.04663039843241374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,3072,0.04157866636912028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,2560,0.039561601479848225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,2560,0.040185598532358806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,7168,0.15298666954040527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,2048,0.03235519925753276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,2048,0.03340693314870198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,6144,0.13052373727162678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,1536,0.02550400098164876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,1536,0.02882453401883443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,8192,0.17606080373128255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,1024,0.018480000893274943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,1024,0.02460800011952718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,4096,0.08883306980133057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,768,0.014478933811187745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,768,0.022167466084162393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,5120,0.10997333526611328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,512,0.01135040024916331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,512,0.02104640007019043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,3584,0.07799893220265706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,256,0.00781333347161611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,3072,0.06603840192159018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,256,0.019976532459259032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,2560,0.05652693510055542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,128,0.007613866527875264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,128,0.018989866971969603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,1536,0.03599679867426554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,64,0.006687999765078227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,10240,32,0.007075199981530507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,64,0.019324799378712974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,10240,32,0.01959999998410543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,2048,0.04628586769104004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,65536,0.48219734827677413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,1024,0.0253930668036143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,65536,0.7195679982503255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,51200,0.38427626291910805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,51200,0.5446890513102214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,16384,0.179911470413208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,16384,0.1454314708709717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,12288,0.13456212679545085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,12288,0.10454933643341065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,256,0.013225600123405457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,512,0.016570666432380678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,128,0.011878400047620138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,10240,0.08560106754302979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,8192,0.0911466677983602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,8192,0.0718943993250529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,10240,768,0.02066453297932943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,7168,0.0800000031789144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,7168,0.06406720081965128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,10240,0.11373013655344646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,6144,0.0698752005894979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,6144,0.05801173448562622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,12288,0.18726293245951336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,5120,0.05796586672465006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,5120,0.05152213176091512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,65536,1.0078997294108072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,4096,0.05028693278630575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,4096,0.045099735260009766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,16384,0.25768960316975914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,3584,0.04277653296788533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,3584,0.04127039909362793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,51200,0.7789909362792968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,3072,0.0416320006052653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,3072,0.03935786485671997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,10240,0.1622154712677002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,2560,0.031752532720565795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,6144,0.10392320156097412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,2560,0.03353386720021566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,2048,0.026369067033131917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,2048,0.030025599400202434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,7168,0.12129173278808594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,1536,0.020447999238967896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,1536,0.026655999819437663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,8192,0.1376255989074707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,1024,0.015631999572118124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,1024,0.024065067370732628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,3072,0.05295679966608683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,768,0.013358933726946512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,768,0.0226197342077891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,4096,0.06932266553243002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,512,0.010708266496658325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,512,0.020460800329844157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,5120,0.08637653191884359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,256,0.0075765331586201985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,256,0.01839359998703003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,3584,0.06134613355000814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,128,0.006363733112812043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,128,0.017604267597198485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,1536,0.02874559958775838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,64,0.00553173323472341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,64,0.0176746666431427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,8192,32,0.005789866546789805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,8192,32,0.01798186699549357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,2560,0.04558826684951782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,65536,0.685649045308431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,65536,0.4735744158426921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,2048,0.037402665615081786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,51200,0.5292842547098796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,51200,0.37251946131388347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,16384,0.17140906651814777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,16384,0.12945173581441244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,256,0.010974933703740437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,12288,0.12697707017262777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,12288,0.09546559651692708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,512,0.014138666788736978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,10240,0.10744106769561768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,10240,0.08327786922454834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,128,0.010021332899729412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,8192,0.08784106572469076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,768,0.017770665884017944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,8192,0.06985493501027426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,7168,0.07737600008646647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,7168,0.06293013493220011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,8192,1024,0.020961066087086998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,6144,0.06636373202006021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,6144,0.05691839853922526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,65536,0.9409749348958334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,5120,0.05939626693725586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,5120,0.0506880005200704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,12288,0.17683413823445637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,4096,0.04479573170344035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,4096,0.04397439956665039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,16384,0.23914453188578286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,3584,0.04086720148722331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,3584,0.03982933362325032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,51200,0.7001120249430339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,3072,0.038924801349639895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,3072,0.0382912000020345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,10240,0.15067520141601562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,2560,0.030130134026209517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,2560,0.03274346590042114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,6144,0.09697706699371338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,2048,0.025050665934880572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,2048,0.03035413424173991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,7168,0.11357226371765136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,1536,0.020472532510757445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,1536,0.026710400978724163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,8192,0.12810560067494708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,1024,0.015575466553370157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,1024,0.023721599578857423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,3072,0.049951998392740886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,768,0.013118933637936911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,768,0.021966934204101562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,4096,0.06562773386637369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,512,0.0102783997853597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,512,0.019997866948445638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,5120,0.08233493169148763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,256,0.007403733332951863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,256,0.018451199928919474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,1536,0.027591466903686523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,2560,0.04177920023600261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,128,0.006142933170000712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,3584,0.058982400099436436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,64,0.007534933090209961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,128,0.020462934176127115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,7168,32,0.008076799909273784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,64,0.021294933557510377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,7168,32,0.020660267273585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,65536,0.41031252543131513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,65536,0.5603733062744141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,2048,0.035709865887959796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,51200,0.45630826950073244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,51200,0.3260320027669271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,1024,0.01997440059979757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,16384,0.140446933110555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,16384,0.11138453483581542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,12288,0.10885120232899984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,51200,0.5539018630981445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,12288,0.08878506819407145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,10240,0.0914517323176066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,10240,0.07641920248667398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,10240,0.11869653065999348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,8192,0.07497173150380453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,8192,0.06358400185902914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,256,0.011061333616574605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,128,0.009941333532333374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,7168,0.06463359991709391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,512,0.013176533579826354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,7168,0.05738879839579264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,6144,0.056995201110839847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,7168,0.08879253069559732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,6144,0.05135999917984009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,5120,0.04816106557846069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,7168,768,0.01698346734046936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,5120,0.04568213224411011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,5120,0.06150506734848023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,4096,0.03838293155034383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,4096,0.04994453191757202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,4096,0.0399455984433492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,3584,0.034439468383789064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,3584,0.036660265922546384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,3072,0.030771199862162275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,3072,0.032765867312749226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,2560,0.026718932390213012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,2560,0.030051199595133464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,12288,0.1374677340189616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,2048,0.0214464008808136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,2048,0.027426133553187054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,2048,0.026998400688171387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,1536,0.01763520042101542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,1536,0.024886399507522583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,16384,0.17752639452616376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,1024,0.013243732849756875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,1024,0.02215999960899353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,1024,0.016528000434239708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,768,0.011045333743095399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,768,0.020142932732899986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,65536,0.7165258407592774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,512,0.009061333537101746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,512,0.019041067361831664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,6144,0.07345279852549234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,256,0.006592000027497609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,256,0.018978132804234823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,256,0.00976746678352356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,128,0.005570133527119955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,128,0.017930666605631508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,128,0.008504533767700195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,64,0.005430399874846141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,64,0.01776533325513204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,6144,32,0.0056415999929110205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,6144,32,0.017812265952428182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,65536,0.45138025283813477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,3072,0.03844053347905477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,65536,0.36482772827148435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,51200,0.3529120127360026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,51200,0.29589974085489906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,2560,0.033319467306137086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,8192,0.10318186283111572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,16384,0.11861120065053303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,16384,0.09596479733784993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,12288,0.08792213598887125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,12288,0.0756341298421224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,3584,0.04458986520767212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,10240,0.07503039836883545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,10240,0.06594773530960082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,65536,0.6516714731852213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,8192,0.06086186567942301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,8192,0.05764266649881998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,16384,0.1632863998413086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,7168,0.05397866566975912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,512,0.011379200220108032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,7168,0.051718401908874514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,6144,0.046648534138997395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,6144,0.046787198384602866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,12288,0.12851520379384357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,5120,0.03846933444341023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,5120,0.0412992000579834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,1536,0.021893332401911415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,6144,768,0.014197333653767904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,4096,0.03192639946937561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,4096,0.0353333314259847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,4096,0.04715413252512614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,3584,0.028518400589625043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,3584,0.03243199984232585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,3072,0.02505279978116353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,3072,0.030138667424519854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,10240,0.1083189328511556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,2560,0.021566933393478392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,51200,0.5040746688842773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,2560,0.02770026723543803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,2048,0.017847466468811034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,2048,0.025458133220672606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,8192,0.09054933389027914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,1536,0.014922666549682616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,1536,0.02340373396873474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,1536,0.020054399967193604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,1024,0.011545600493748982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,1024,0.020536533991495767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,7168,0.07911787033081055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,768,0.009798399607340495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,6144,0.06989759604136149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,768,0.019568000237147012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,512,0.0077674667040507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,512,0.01849386692047119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,5120,0.05914133389790853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,3584,0.04144853353500366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,256,0.00588266650835673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,256,0.01804373264312744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,3072,0.03614826599756877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,128,0.00552106648683548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,2560,0.030562132596969604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,128,0.0169706662495931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,64,0.005148800214131674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,64,0.016876800855000814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,2048,0.024486400683720908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,5120,32,0.0047765334447224935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,5120,32,0.017119999726613364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,65536,0.3710890769958496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,65536,0.32901118596394857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,51200,0.2927178700764974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,51200,0.2623893260955811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,1024,0.015579733252525329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,16384,0.0935103972752889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,16384,0.08384746710459391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,768,0.01285653313000997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,12288,0.07224319775899252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,12288,0.0683690627415975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,512,0.010761599739392598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,10240,0.0590837319691976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,10240,0.05847893158594767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,8192,0.047440000375111896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,256,0.008985599875450135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,8192,0.05051519870758057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,7168,0.041891201337178545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,7168,0.045661866664886475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,5120,128,0.008140799899895985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,6144,0.03626559972763062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,6144,0.04103786547978719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,65536,0.5887317021687826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,12288,0.11756479740142822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,5120,0.032485334078470866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,5120,0.03642773230870565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,4096,0.02601813276608785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,4096,0.032103466987609866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,16384,0.15472960472106934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,3584,0.02286293307940165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,3584,0.029000532627105714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,6144,0.0641696015993754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,3072,0.019695999224980672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,3072,0.027204267183939618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,10240,0.1041984001795451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,51200,0.46674559911092117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,2560,0.01697493394215902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,2560,0.02534719904263814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,2048,0.014574933052062988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,2048,0.02405973275502523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,7168,0.07512426376342773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,1536,0.012058666348457337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,1536,0.02153493364651998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,8192,0.08593066533406576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,1024,0.009655466675758362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,1024,0.018820265928904213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,3072,0.03375680049260457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,768,0.008028799792130788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,768,0.01880853374799093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,4096,0.045203200976053876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,512,0.00673280010620753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,1536,0.018294399976730345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,512,0.017501866817474364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,256,0.005312000215053558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,256,0.016509866714477538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,2560,0.02868693272272746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,5120,0.0554197351137797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,128,0.004529066880544027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,128,0.016294399897257485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,64,0.004517333209514618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,64,0.016402133305867515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,4096,32,0.004633600016434988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,4096,32,0.0164000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,3584,0.03969920078913371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,65536,0.3339669227600098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,2048,0.02323733369509379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,65536,0.3194954554239909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,1024,0.014249599973360696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,51200,0.2648970603942871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,51200,0.24444479942321778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,16384,0.0908682664235433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,16384,0.08200853665669759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,12288,0.06875733534495035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,12288,0.0649845321973165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,128,0.007190399865309398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,10240,0.056745600700378415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,10240,0.05689386526743571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,512,0.009334400296211243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,8192,0.04593706528345744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,8192,0.049721598625183105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,768,0.011638399958610535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,7168,0.04053973356882731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,7168,0.045043198267618816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,4096,256,0.008113066852092742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,6144,0.03618026574452718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,6144,0.04043200016021729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,12288,0.11061440308888752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,5120,0.03030293385187785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,16384,0.14520427385965984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,65536,0.5482933044433593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,51200,0.43437334696451824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,4096,0.025268266598383587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,4096,0.0312554657459259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,5120,0.035580801963806155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,3584,0.02232853372891744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,3584,0.028775467475255327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,3072,0.01951893369356791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,3072,0.026945066452026368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,10240,0.09921279748280844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,2560,0.016642133394877114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,2560,0.025225599606831867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,6144,0.06129493316014608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,2048,0.014455466469128927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,2048,0.023379200696945192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,7168,0.07236053148905436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,1536,0.011945600310961407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,1536,0.02177600065867106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,8192,0.07970346609751383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,1024,0.009406933188438415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,1024,0.01952426632245382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,4096,0.042105599244435625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,5120,0.05229013363520304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,768,0.008175999919573466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,768,0.018675200144449868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,512,0.006483200192451477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,512,0.017962666352589925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,3072,0.031249066193898518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,256,0.005312000215053558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,256,0.016860800981521606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,3584,0.03713493347167969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,128,0.004594133297602335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,128,0.0167797327041626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,1536,0.017758933703104655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,64,0.004716800153255462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,64,0.017042134205500284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3584,32,0.0049216002225875854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3584,32,0.01722773313522339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,2560,0.028066132465998334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,65536,0.28777599334716797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,65536,0.27189652125040686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,1024,0.013310933113098144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,51200,0.22961066563924154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,65536,0.4244384129842122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,16384,0.07242453098297119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,51200,0.21499093373616537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,16384,0.076146133740743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,12288,0.054357333978017175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,51200,0.33323520024617515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,16384,0.10958826541900635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,12288,0.05998293161392212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,10240,0.04590186675389608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,2048,0.022759467363357544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,12288,0.08487359682718912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,10240,0.052215464909871424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,10240,0.07007359663645427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,8192,0.04415040016174317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,7168,0.033445334434509276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,8192,0.05819840033849081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,7168,0.03971519867579142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,6144,0.02860906720161438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,7168,0.05030719836552938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,6144,0.03600533405939738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,5120,0.024629332621892295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,6144,0.04495786825815837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,4096,0.020061866442362467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,5120,0.03126613299051921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,4096,0.028493867317835493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,3584,0.018382932742436728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,3584,0.026920533180236815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,3584,0.02831253409385681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,3072,0.016165332992871602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,3072,0.025194666783014935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,8192,0.03805973529815674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,2560,0.014145066340764364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,2560,0.023640533288319908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,2560,0.020933334032694498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,5120,0.036804266770680744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,2048,0.012121599912643433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,2048,0.022076799472173055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,1536,0.010001066327095031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,1536,0.019985065857569376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,1536,0.0136245330174764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,1024,0.007899733384450276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,512,0.00944533348083496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,1024,0.018272000551223754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,768,0.006660266717274983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,1024,0.011062399546305338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,256,0.00810346653064092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,768,0.00960106650988261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,768,0.01752106746037801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,512,0.005666133264700572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,512,0.017336533466974894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,256,0.004619733492533366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,256,0.016721065839131674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,128,0.007313066720962524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,128,0.004565333326657614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3584,768,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,128,0.016130133469899496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,128,0.006513066589832306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,64,0.004113066693147024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,64,0.016246400276819863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,3072,32,0.004156800111134847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,3072,32,0.015890133380889893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,65536,0.23834773699442544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,4096,0.0309173325697581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,65536,0.2600074609120687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,51200,0.2085909366607666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,51200,0.2942048072814941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,256,0.0070165331164995836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,16384,0.06292693217595419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,16384,0.07018773555755616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,12288,0.04785173336664836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,12288,0.05691413482030233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,12288,0.07702399889628092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,10240,0.040584532419840495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,10240,0.04976426760355632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,51200,0.18750613530476887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,8192,0.033508265018463136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,8192,0.041832534472147624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,7168,0.029475200176239013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,7168,0.03776959975560506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,512,0.008254933357238769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,6144,0.02563199996948242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,6144,0.03368213176727295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,3072,0.02378986676534017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,5120,0.021934932470321654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,5120,0.030264532566070555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,5120,0.03622506856918335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,4096,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,4096,0.027113600571950273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,4096,0.029028266668319702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,3584,0.015995732943216958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,3584,0.02574186722437541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,65536,0.3866154670715332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,3072,2048,0.017543466885884602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,3072,0.014114133516947427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,3072,0.02376533349355062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,2560,0.012567466497421265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,2560,0.022898133595784506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,16384,0.10169280370076497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,2048,0.010737066467603047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,2048,0.021661865711212158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,2048,0.016355199615160625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,1536,0.00906986693541209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,1536,0.019483733177185058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,10240,0.06560746828715006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,1024,0.007192533214886982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,1024,0.018028799692789713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,1024,0.01064639985561371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,768,0.006211199859778086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,768,0.017678932348887125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,8192,0.05589439868927002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,512,0.0052821333209673565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,512,0.01688533425331116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,512,0.007930666704972585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,256,0.004489600161711375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,256,0.015878400206565856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,256,0.006816000243028005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,128,0.003929600119590759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,128,0.015821866194407144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,7168,0.048395733038584396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,6144,0.04102613528569539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,64,0.0038805333276589714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2560,32,0.004120533416668574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,64,0.015925332903862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2560,32,0.01581439971923828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,65536,0.19029547373453776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,65536,0.23392747243245443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,51200,0.15387627283732097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,51200,0.1857653299967448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,51200,0.26964588165283204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,16384,0.07100266615549723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,16384,0.0731381336847941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,3584,0.026700800657272337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,12288,0.054382932186126706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,12288,0.05993386507034302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,128,0.006171733140945435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,65536,0.3482975959777832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,10240,0.0493343989054362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,3072,0.022526933749516805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,10240,0.05221226612726847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,8192,0.03580586512883504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,8192,0.041665065288543704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,8192,0.05117013454437256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,1536,0.013035733501116434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,7168,0.03261546691258748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,7168,0.03783359924952189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,6144,0.028598399957021077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,6144,0.03464106718699138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,5120,0.02537279923756917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,5120,0.031658667325973514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,768,0.009044266740481059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,4096,0.019512534141540527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,4096,0.02758079965909322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,16384,0.09560213088989258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,3584,0.018182400862375894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,3584,0.02595733404159546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2560,2560,0.019666133324305217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,12288,0.07089599768320719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,3072,0.016006400187810264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,3072,0.02442453304926554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,10240,0.06195520162582398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,3072,0.0210207998752594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,2560,0.014019200205802917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,2560,0.022106667359670006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,2048,0.011528533697128297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,2048,0.020911999543507895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,7168,0.04504106839497884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,1536,0.009388800462086995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,6144,0.03752426703770955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,1536,0.012065066893895467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,1536,0.019399466117223103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,1024,0.007286400099595388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,1024,0.018282665808995565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,768,0.006341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,5120,0.03294186592102051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,768,0.017522132396697997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,512,0.005162666738033295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,512,0.016597333550453185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,4096,0.02694293260574341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,256,0.004136533290147781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,256,0.016337066888809204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,3584,0.02493013342221578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,128,0.003995733211437861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,128,0.015421866377194723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,64,0.003605333218971888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,64,0.015453867117563882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,2048,32,0.0036821333070596062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,2048,32,0.015530666708946228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,65536,0.14969813028971354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,65536,0.20906559626261392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,2560,0.018169599771499633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,2048,0.015218133727709452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,51200,0.11727999846140544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,1024,0.010036266843477885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,51200,0.1665162722269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,768,0.008505599697430928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,16384,0.05973866780598959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,12288,0.030689066648483275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,512,0.007227733234564463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,12288,0.04773759841918945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,10240,0.02658240000406901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,10240,0.040642134348551434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,256,0.006235733131567637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,8192,0.022907733917236328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,8192,0.03394453525543213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,2048,128,0.005662933488686879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,7168,0.02107093334197998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,7168,0.031113600730895995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,16384,0.04228693246841431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,6144,0.016318933169047038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,6144,0.02887679934501648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,65536,0.30997225443522136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,5120,0.014506666858990987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,5120,0.0264682670434316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,51200,0.2515552043914795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,16384,0.08116693496704101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,4096,0.012044800321261089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,12288,0.06196800072987875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,4096,0.023874133825302124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,3584,0.010943999886512757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,3584,0.022338134050369263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,3072,0.010177066922187806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,3072,0.020721065998077392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,10240,0.05346773465474447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,2560,0.008993066350618998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,2560,0.01998293399810791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,8192,0.042743468284606935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,2048,0.007894399762153625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,2048,0.01910933256149292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,7168,0.03700799942016601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,1536,0.006532266736030579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,1536,0.018233599265416463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,6144,0.03309973279635112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,1024,0.005423999826113383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,1024,0.017439999183019004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,5120,0.028331732749938963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,768,0.00492799977461497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,768,0.016809600591659545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,4096,0.02324906587600708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,512,0.004324266811211904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,3072,0.018334933121999106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,512,0.016484266519546507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,256,0.00411520004272461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,2048,0.013385599851608277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,256,0.015553067127863566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,128,0.003566933423280716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,128,0.015544533729553223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,1024,0.009201066692670186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,64,0.003419733295838038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,64,0.01548799971739451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1536,32,0.003583999971548716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1536,32,0.015432533621788026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,65536,0.101910400390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,3584,0.02134933272997538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,2560,0.01622293293476105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,65536,0.18737813631693523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,51200,0.08072426319122314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,51200,0.1504458745320638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,1536,0.011678933103879293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,16384,0.032491733630498246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,16384,0.05676586627960205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,768,0.008106666803359985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,12288,0.02672213315963745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,12288,0.04652693271636963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,128,0.0058677335580190025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,10240,0.02323413292566935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,10240,0.03921706676483154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,8192,0.018935465812683107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,256,0.006486399968465169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,8192,0.03379520177841187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,7168,0.020562134186426797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,7168,0.0315775990486145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1536,512,0.007253333429495494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,6144,0.01798400084177653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,6144,0.028886399666468304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,65536,0.2893312136332194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,5120,0.015659733613332113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,5120,0.02587839961051941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,51200,0.22727680206298828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,4096,0.013292800386746725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,4096,0.02481493353843689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,16384,0.08108479976654052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,3584,0.011760000387827556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,3584,0.02190933426221212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,12288,0.06107946634292603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,3072,0.010714667042096455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,3072,0.02068693240483602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,10240,0.0503104011217753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,2560,0.010288000106811523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,2560,0.02058026591936747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,8192,0.04335466623306274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,2048,0.008178133269151051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,7168,0.039033599694569904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,2048,0.020091732343037925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,1536,0.006877866884072621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,1536,0.018617600202560425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,6144,0.03313386638959249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,1024,0.005738666653633118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,1024,0.017222400506337485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,4096,0.02230613430341085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,768,0.005103999873002371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,5120,0.028963200251261395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,768,0.016800000270207723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,512,0.004410666724046072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,512,0.016145066420237223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,3584,0.02095680038134257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,256,0.0037920000652472176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,256,0.01548373301823934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,3072,0.018193066120147705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,128,0.0033461332321166994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,128,0.01516480048497518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,1536,0.010777599612871806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,64,0.003052799900372823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,2560,0.015607466300328573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,64,0.015194666385650635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,1024,32,0.0032448001205921174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,1024,32,0.015014400084813436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,65536,0.08391146659851074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,65536,0.17670399347941082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,51200,0.06644800106684366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,2048,0.013100799918174744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,51200,0.1423914591471354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,16384,0.027031467358271284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,1024,0.008890666564305623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,16384,0.05200426578521729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,12288,0.02175359924634298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,12288,0.042294398943583174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,512,0.006611200173695882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,256,0.005905066430568695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,10240,0.018931200106938682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,10240,0.037855998675028486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,768,0.007548800110816956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,8192,0.01578879952430725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,8192,0.03158400058746338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,7168,0.016583466529846193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,7168,0.029290666182835896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,1024,128,0.0053269331653912864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,6144,0.014719999829928079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,6144,0.02675093412399292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,65536,0.27438294092814125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,5120,0.012921599547068277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,5120,0.026286933819452924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,51200,0.20732266108194985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,4096,0.011062399546305338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,4096,0.024370133876800537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,16384,0.07206292947133383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,3584,0.010193066795667012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,3584,0.022348799308141074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,12288,0.05362879832585653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,3072,0.009096533060073853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,3072,0.020307199160257975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,10240,0.04632960160573323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,2560,0.008573866883913676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,8192,0.03816959857940674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,2560,0.019504000743230186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,2048,0.007166933516661327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,2048,0.018307199080785118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,7168,0.03344426552454631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,6144,0.029322665929794312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,1536,0.005955199897289276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,1536,0.01781546672185262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,1024,0.004894933104515076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,1024,0.016822399695714314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,5120,0.025541333357493083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,768,0.004458666841189066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,768,0.016476800044377647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,4096,0.020651733875274657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,512,0.0038634667793909705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,512,0.01602240006128947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,3072,0.016140799721082053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,256,0.0034613333642482757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,256,0.015326933066050211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,3584,0.018946133057276406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,128,0.0031925333042939507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,128,0.015126400192578635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,2048,0.012029866377512615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,64,0.0030975999931494398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,64,0.014912000298500061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,768,32,0.00306986669699351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,768,32,0.014940800269444785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,65536,0.061441067854563394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,65536,0.16639679272969563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,2560,0.014547200004259745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,51200,0.05464853445688883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,51200,0.13512959480285644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,1536,0.010033067067464192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,16384,0.020568533738454183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,16384,0.050012799104054775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,16384,0.06659839948018392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,12288,0.01580586632092794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,12288,0.03855786720911662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,1024,0.008005333443482716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,10240,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,10240,0.03417706489562988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,768,0.0071712002158164975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,8192,0.016081066926320393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,8192,0.02977706591288249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,512,0.006477866570154827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,7168,0.014276267091433207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,7168,0.028066132465998334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,256,0.0056757330894470215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,6144,0.0127018670241038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,65536,0.23144853909810387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,6144,0.026829866568247478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,6144,0.027239465713500978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,5120,0.01128426690896352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,5120,0.023915733893712363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,4096,0.009378133217493693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,51200,0.1872085412343343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,4096,0.02263573408126831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,3584,0.008734933535257975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,3584,0.021193599700927733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,12288,0.048698667685190836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,3072,0.008001066744327545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,3072,0.020119466384251914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,768,128,0.005278933544953664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,2560,0.007201066613197327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,2560,0.01925333340962728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,10240,0.0399829347928365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,2048,0.00647680014371872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,2048,0.018875734011332194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,8192,0.034017066160837814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,1536,0.005549866457780202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,1536,0.017195733388264973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,7168,0.031673600276311234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,1536,0.0090421328941981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,1024,0.00468800018231074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,1024,0.016591999928156534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,768,0.004257066547870636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,768,0.016313599546750386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,5120,0.022951465845108033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,512,0.00386559988061587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,4096,0.019347200791041054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,512,0.015826132893562318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,256,0.003306666761636734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,3584,0.017746132612228394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,256,0.015357866883277893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,128,0.0030613332986831666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,128,0.01511573294798533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,3072,0.015404799580574035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,64,0.0030069333811601003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,64,0.01498240033785502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,512,32,0.0030421334008375804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,512,32,0.014892799655596414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,65536,0.043645866711934406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,2560,0.013566933075586953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,2048,0.010603732864061991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,51200,0.036339199542999266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,51200,0.1248970667521159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,65536,0.15616426467895508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,1024,0.007886933286984761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,16384,0.017218132813771568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,16384,0.042583465576171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,768,0.006884266436100006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,12288,0.013496533036231995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,12288,0.03336639801661174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,512,0.006342400113741558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,10240,0.012437333663304646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,10240,0.03203733364741008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,256,0.005555200080076853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,8192,0.010168533523877461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,512,128,0.005082666873931885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,8192,0.02848213315010071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,7168,0.010419199864069622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,7168,0.026320000489552815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,65536,0.1884874661763509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,6144,0.009623466928799947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,6144,0.02500373323758443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,6144,0.020696532726287842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,5120,0.010203733046849569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,5120,0.02321386734644572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,51200,0.1527989387512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,4096,0.008569600184758504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,4096,0.021447465817133585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,16384,0.05270933310190836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,3584,0.007949866851170858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,3584,0.02119999925295512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,12288,0.04044906695683797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,3072,0.00713919997215271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,3072,0.019735467433929444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,2560,0.00655680000782013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,10240,0.03461653391520182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,2560,0.019644800821940103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,2048,0.005841066439946493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,2560,0.010629333058993022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,2048,0.018180267016092936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,1536,0.005041066805521647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,1536,0.017385600010553996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,8192,0.027560534079869588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,7168,0.024146133661270143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,1024,0.004388266801834106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,1024,0.016356266538302102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,768,0.003980800012747447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,768,0.015945600469907124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,5120,0.018317866325378417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,512,0.003618133316437403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,3584,0.014247467120488485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,512,0.015941333770751954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,256,0.0032320000231266023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,256,0.014756266276041666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,2048,0.009100799759229023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,128,0.002899199972550074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,128,0.014665599664052328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,1536,0.008436266581217449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,64,0.002846933404604594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,64,0.015190399686495461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,256,32,0.002863999952872594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,1024,0.006938666601975759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,256,32,0.014929067095120749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,768,0.006066133578618368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,65536,0.035257601737976076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,4096,0.015803733468055726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,65536,0.150983460744222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,51200,0.12126932938893635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,51200,0.11543893019358317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,16384,0.010256000359853109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,16384,0.039019731680552165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,3072,0.012146133184432983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,16384,0.03966399828592936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,12288,0.00941439966360728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,12288,0.030011733373006184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,12288,0.031138134002685548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,10240,0.009617066383361817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,8192,0.008520533641179402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,10240,0.028614399830500285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,10240,0.025563732782999678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,7168,0.008573866883913676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,8192,0.026358399788538617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,7168,0.024975999196370443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,6144,0.007937066753705342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,6144,0.023630932966868082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,51200,0.029165865977605183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,5120,0.0071946665644645694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,5120,0.02314880092938741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,5120,0.014971733093261719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,4096,0.007158400118350982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,4096,0.021375999848047892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,512,0.005776000022888183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,3584,0.006769066552321117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,256,0.005267199873924255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,3584,0.020345600446065266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,3072,0.006457599997520447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,3072,0.01955946683883667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,256,128,0.00487253318230311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,2560,0.006209066510200501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,2560,0.018539732694625853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,65536,0.14630719820658367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,2048,0.005533866584300995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,2048,0.01761173407236735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,8192,0.021252266565958657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,1536,0.004857600231965383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,7168,0.019170133272806804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,1536,0.01713706652323405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,1024,0.00413973331451416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,1536,0.007942399879296621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,1024,0.016323199868202208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,768,0.0037973334391911825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,6144,0.016822399695714314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,768,0.01597653329372406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,4096,0.012588799993197123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,512,0.0034944000343481696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,512,0.01544533371925354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,3584,0.01209386686484019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,256,0.003094399968783061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,3072,0.010524800419807434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,256,0.01487573285897573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,128,0.0028661333024501802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,128,0.014787200093269347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,2560,0.009604266285896302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,64,0.0026602665583292644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,64,0.01463573376337687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,128,32,0.0027093333502610523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,128,32,0.014362667004267374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,65536,0.03155840039253235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,2048,0.008455466230710347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,51200,0.026357332865397137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,65536,0.14839253425598145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,16384,0.010409599542617798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,51200,0.11925546328226726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,12288,0.008904533584912618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,16384,0.03594133456548055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,12288,0.029785599311192828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,10240,0.0076682666937510175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,8192,0.006915199756622315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,10240,0.02775786717732747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,7168,0.006635733445485433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,8192,0.02476159930229187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,6144,0.00625600020090739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,1024,0.006866133213043213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,7168,0.02456000049908956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,5120,0.00668693333864212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,6144,0.02432960073153178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,5120,0.02256853381792704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,4096,0.006206933160622915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,3584,0.006543999910354615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,4096,0.02110613385836283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,3584,0.020055466890335084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,3072,0.006281599899133046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,2560,0.0062730665008227035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,2048,0.005443199972311656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,3072,0.01904746691385905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,2560,0.01883413394292196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,1536,0.004758400221665701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,768,0.006074666480223338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,2048,0.017644800742467246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,1024,0.0041184000670909885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,1536,0.01694933374722799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,768,0.0037962667644023894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,1024,0.016500266393025716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,256,0.0031530665854612983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,512,0.015422933300336204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,128,0.0029504001140594482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,256,0.014905599753061929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,64,0.002717866748571396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,128,0.014537599682807923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,32,0.0027232001225153605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,64,0.014729600151379904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,65536,0.032257066170374556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,32,0.014490666985511779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,51200,0.026315732796986895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,16384,0.010039466619491576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,65536,0.14761172930399577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,51200,0.11805013020833333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,12288,0.007886933286984761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,16384,0.03562560081481934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,10240,0.007643733421961467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,12288,0.029772800207138062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,8192,0.0071733335653940845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,7168,0.006626133124033611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,8192,0.024822400013605753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,6144,0.007005866865317028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,7168,0.024576000372568765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,6144,0.02281493345896403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,5120,0.006593066453933716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,4096,0.0061589335401852924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,512,0.005657599866390228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,5120,0.02244159976641337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,3584,0.006554666658242543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,4096,0.02108373244603475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,3072,0.006221866607666016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,3584,0.01953386664390564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,2560,0.006151466568311056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,3072,0.019180800517400107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,2048,0.0054058666030565895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,1536,0.0048213332891464235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,2048,0.017874133586883546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,1024,0.004102399945259095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,1536,0.017017600933710735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,768,0.003828266759713491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,1024,0.016210132837295534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,512,0.0034143999218940735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,256,0.005193600058555603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,768,0.01607039968172709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,512,0.015910399953524272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,64,512,0.0034656000634034475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,256,0.01518186628818512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,128,0.0029056000212828318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,64,768,0.016049066185951234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,64,0.0026986666023731233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,128,0.014754133423169455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,32,0.0027295999228954316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,513,128,128,0.004870399832725525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,64,0.014588800072669984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,32,0.014719999829928079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,10240,0.027355732520421346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,513,32,2560,0.018237866957982383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,513,32,256,0.0030954666435718536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,16384,0.7616352081298828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,16384,1.424456532796224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,12288,0.5879711786905925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,12288,1.0657866795857749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,10240,0.4759669303894043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,51200,2.4299476623535154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,10240,0.9038080215454102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,8192,0.4073898633321126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,8192,0.6971370697021484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,7168,0.35386133193969727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,7168,0.6146165211995442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,51200,4.735547892252604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,6144,0.538378651936849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,16384,1.1021291097005208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,12288,0.8218112309773764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,6144,0.31337067286173503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,5120,0.47856000264485676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,5120,0.2633685270945231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,10240,0.7095701217651367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,4096,0.37264105478922527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,4096,0.21622932751973473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,3584,0.3190656026204427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,4096,0.27948694229125975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,3584,0.1938602606455485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,3072,0.16741867065429689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,3072,0.27558720906575523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,8192,0.5348810831705729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,2560,0.23277546564737955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,2560,0.14538346926371257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,2048,0.1865440050760905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,2048,0.12104640007019044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,7168,0.4692373275756836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,1536,0.16023680369059246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,51200,4.681783548990885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,1536,0.11237760384877522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,1536,0.12410773436228435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,1024,0.10309546788533527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,6144,0.41719681421915694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,1024,0.07987093130747477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,768,0.08782080014546713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,5120,0.344976011912028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,768,0.08127573331197104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,512,0.061843200524648034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,512,0.0582261323928833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,256,0.044222935040791826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,256,0.050529066721598306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,3584,0.24401386578877768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,128,0.036415998140970865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,128,0.044691201051076254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,3072,0.21171199480692543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,128,0.05178133249282837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,64,0.03711146513621012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,32,0.031883732477823896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,64,0.04585386514663696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,768,0.06766613324483237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,32,0.04804799954096477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,2560,0.17988373438517252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,2048,0.13939414024353028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,1024,0.08900159994761149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,512,0.06554133494695027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,256,0.055905067920684816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,51200,2.005146662394206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,16384,1.216101328531901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,65536,2.795025126139323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,12288,0.9329546610514322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,51200,3.9315521240234377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,12288,0.5074048042297363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,16384,0.613258679707845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,10240,0.409606393178304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,65536,4.869751485188802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,10240,0.7550602595011393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,8192,0.5860629399617513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,8192,0.33940585454305017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,7168,0.5436373392740885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,7168,0.30533227920532224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,6144,0.4282848040262858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,6144,0.25603413581848145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,12288,0.6101418813069661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,5120,0.36420907974243166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,16384,0.8127904256184897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,5120,0.23745279312133788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,65536,3.689555104573568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,4096,0.2920138676961263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,4096,0.19895893732706707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,3584,0.26732587814331055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,10240,0.5389567693074544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,3584,0.16436160405476888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,3072,0.22630613644917807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,3072,0.14839359919230144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,51200,2.9024192810058596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,6144,0.30972798665364587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,2560,0.1903114636739095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,2560,0.12309973239898682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,7168,0.37473173141479493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,2048,0.16047679583231608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,2048,0.11096853415171307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,8192,0.42324479420979816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,1536,0.1237557331720988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,1536,0.09224960009256998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,1024,0.09245973428090414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,1024,0.07156799634297689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,3072,0.1613034725189209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,768,0.07486186822255453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,768,0.05924586852391561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,4096,0.21405973434448242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,512,0.05420480171839396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,512,0.05115840037663778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,1536,0.09017386436462402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,256,0.03654613494873047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,5120,0.26386879285176595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,256,0.04370666742324829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,128,0.029945600032806396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,128,0.040194133917490646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,2560,0.13989653587341308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,64,0.02940906683603923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,3584,0.19044586817423503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,64,0.038867199420928956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,51200,32,0.025858134031295776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,51200,32,0.04114026625951131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,65536,0.8365877151489258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,2048,0.12158933480580647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,65536,1.4292437235514321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,51200,1.1349866231282553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,1024,0.07090240319569906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,51200,0.6586389541625977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,65536,1.123349380493164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,16384,0.3431370735168457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,16384,0.22203200658162436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,16384,0.2643167972564697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,12288,0.16722240447998046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,12288,0.26173226038614905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,10240,0.15627412796020507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,10240,0.21941013336181642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,8192,0.17690773010253907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,10240,0.17729066212972006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,8192,0.1175434668858846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,7168,0.1555967966715495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,7168,0.10724906921386719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,128,0.03410880168279012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,256,0.03921066522598267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,6144,0.13497066497802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,6144,0.08976533412933349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,5120,0.11077760060628254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,5120,0.07774293422698975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,512,0.045909333229064944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,5120,0.08718720277150473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,4096,0.09138986269632975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,4096,0.06554773251215616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,3584,0.07983466784159342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,3584,0.059876267115275064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,51200,768,0.05603520075480143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,3072,0.07029653390248616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,3072,0.05417173306147257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,3072,0.05406933228174845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,2560,0.059476268291473386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,2560,0.04823466539382935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,12288,0.21015146573384605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,2048,0.04817706743876139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,2048,0.04327573378880818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,2048,0.03999253511428833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,51200,0.9048149108886718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,1536,0.03781546751658122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,1536,0.034168533484141034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,1536,0.04527039925257365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,1024,0.027663999795913698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,8192,0.13361387252807616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,1024,0.03701440095901489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,768,0.022962133089701332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,768,0.033589335282643636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,512,0.01870186726252238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,512,0.027194666862487792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,7168,0.1160703976949056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,256,0.012453333536783854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,256,0.02249173323313395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,6144,0.10529173215230306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,128,0.009401599566141765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,128,0.019747199614842732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,768,0.024317866563796996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,4096,0.06999359925587972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,64,0.007659733295440674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,32,0.008172800143559773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,64,0.020423465967178346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,32,0.02065599958101908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,3584,0.0626698652903239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,2560,0.04767466783523559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,65536,0.6715050379435221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,65536,1.1039413452148437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,1024,0.027054933706919353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,51200,0.9003925323486328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,512,0.02044373353322347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,16384,0.26816320419311523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,51200,0.5367455800374349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,12288,0.20277226765950523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,256,0.01755839983622233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,12288,0.15103893280029296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,10240,0.16954666773478191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,128,0.01577279965082804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,10240,0.12168746789296467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,8192,0.13648212750752767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,8192,0.09880639712015787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,16384,0.21742827097574868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,7168,0.12102080186208089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,7168,0.08523946603139242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,6144,0.10622933705647786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,6144,0.07616746425628662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,51200,0.6780853271484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,5120,0.09052266279856364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,5120,0.06766613324483237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,65536,0.8320159912109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,4096,0.07731306552886963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,10240,0.13818346659342448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,4096,0.06334933439890543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,3584,0.0679253339767456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,3584,0.057714132467905674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,12288,0.15954772631327313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,3072,0.05422293345133463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,3072,0.04542933305104573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,16384,0.20573973655700684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,2560,0.04565226634343465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,2560,0.04077333211898804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,7168,0.0890122652053833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,2048,0.03816320101420085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,2048,0.03678826491038005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,8192,0.10111467043558757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,1536,0.02951573332150777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,1536,0.03191253344217936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,6144,0.0787893295288086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,1024,0.021539199352264404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,1024,0.027037866910298664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,5120,0.06556586821873983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,768,0.017205333709716795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,768,0.025117866198221844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,2560,0.03699733416239421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,512,0.01341973344484965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,512,0.02272319992383321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,4096,0.0530240019162496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,256,0.00974720021088918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,256,0.020132267475128175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,3584,0.04836479822794597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,128,0.007315200070540111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,128,0.018649599949518838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,2048,0.0313045342763265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,64,0.006371200084686279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,64,0.019008000691731773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,3072,0.04121066729227702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,32,0.0067221333583196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,32,0.019308799505233766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,65536,0.5739039738972982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,1536,0.026602667570114136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,65536,0.9057621637980142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,51200,0.6725002924601238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,51200,0.5066442807515462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,16384,0.2207690715789795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,16384,0.15774292945861818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,1024,0.021022933721542358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,12288,0.1696191946665446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,12288,0.1187872012456258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,128,0.012802132964134216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,256,0.013822933038075766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,10240,0.13719360033671063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,10240,0.1005621353785197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,768,0.019093332688013713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,512,0.016270933548609416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,8192,0.11320319970448811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,8192,0.08381866614023845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,7168,0.10047679742177326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,7168,0.0757589340209961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,6144,0.08635306358337402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,65536,0.7160128275553386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,6144,0.06749973297119141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,5120,0.07281920115152994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,5120,0.05890239874521891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,51200,0.6042741139729817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,16384,0.1765621344248454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,4096,0.06581120093663534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,4096,0.05041386683781942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,3584,0.052805332342783604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,3584,0.04679679870605469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,3584,0.04402773380279541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,3072,0.04598933458328247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,3072,0.04237333138783773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,12288,0.13624533017476398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,2560,0.039768532911936445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,2560,0.03857280015945434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,10240,0.12366613547007244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,2048,0.03205546736717224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,8192,0.09040319919586182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,2048,0.03439040184020996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,1536,0.026502400636672974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,1536,0.029365332921346028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,7168,0.08145706653594971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,6144,0.07109546661376953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,1024,0.01871573328971863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,1024,0.025986133019129436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,5120,0.05923733313878378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,768,0.015468800067901611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,768,0.02413866718610128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,4096,0.04972373247146607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,512,0.012335999806722005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,512,0.022311466932296752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,3072,0.037649067242940266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,256,0.009283199906349182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,256,0.01948053240776062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,2560,0.03484373490015666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,128,0.007101866602897644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,2048,0.028702932596206664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,1536,0.023720532655715942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,64,0.0063285330931345625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,64,0.018550399939219156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,32,0.0065194666385650635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,32,0.018641066551208497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,1024,0.019849600394566853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,768,0.016528000434239708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,65536,0.4795680046081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,65536,0.6967231750488281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,128,0.018603734175364175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,512,0.01444906691710154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,51200,0.5479189554850261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,16384,0.17752960522969563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,16384,0.12388799985249836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,51200,0.3771199862162272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,12288,0.13484692573547363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,12288,0.09715627034505209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,10240,0.11256213188171386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,256,0.0116757333278656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,10240,0.08394239743550619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,8192,0.08976213137308756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,8192,0.0701141357421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,7168,0.07785600026448568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,7168,0.06238720019658407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,128,0.010436266660690308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,6144,0.06703466574350993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,6144,0.05623466571172079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,65536,0.5806357065836589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,5120,0.05858560005823771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,5120,0.05245333512624105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,16384,0.14238932927449544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,51200,0.46606400807698567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,4096,0.04677120049794515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,12288,0.10970880190531414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,4096,0.04614400068918864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,3584,0.0415669322013855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,10240,0.09185386498769124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,3584,0.0414741317431132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,3072,0.036755200227101645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,8192,0.06602986653645834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,3072,0.03735893170038859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,2560,0.031016532580057782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,2560,0.033539199829101564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,2048,0.026183466116587322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,7168,0.05899519920349121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,2048,0.03089066743850708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,1536,0.02087786595026652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,1536,0.026858667532602947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,6144,0.05488640069961548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,1024,0.0150026669104894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,1024,0.023269333442052207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,5120,0.04544853369394938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,768,0.012932266791661581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,768,0.02227413256963094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,4096,0.03779413302739461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,512,0.010333866874376933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,512,0.02004800041516622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,3072,0.029513599475224812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,256,0.007464533547560374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,2560,0.025885866085688276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,256,0.0184661328792572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,256,0.010541866223017376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,128,0.006161066889762879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,2048,0.022612265745798745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,128,0.017760000626246133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,64,0.005508266886075338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,32,0.005799466868241628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,64,0.017539199193318686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,32,0.017836799224217735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,1536,0.019655466079711914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,65536,0.45200214385986326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,65536,0.6422815958658854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,3584,0.034355199337005614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,1024,0.015939199924468996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,51200,0.49534400304158527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,51200,0.3670581181844076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,16384,0.160807466506958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,16384,0.12089172999064128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,12288,0.12189866701761883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,12288,0.09477972984313965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,10240,0.10286186536153157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,10240,0.08135253588358561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,768,0.01469013293584188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,8192,0.08296639919281006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,512,0.012500266234079996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,8192,0.06813866297403971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,128,0.009983999530474345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,7168,0.07455573081970215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,7168,0.06242666641871134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,65536,0.47896318435668944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,6144,0.0627072016398112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,6144,0.06224853197733561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,51200,0.4069696108500163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,16384,0.12525760332743327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,5120,0.055078399181365964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,5120,0.04890880187352498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,12288,0.09360853036244711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,4096,0.04176213343938191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,4096,0.039426132043202715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,3584,0.03651306629180908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,10240,0.07887146472930909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,3584,0.03644479910532634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,3072,0.03146026730537414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,3072,0.03317546645800273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,2560,0.026818132400512694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,2560,0.030460800727208453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,8192,0.06603519916534424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,2048,0.022424532969792684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,2048,0.027292799949645997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,2048,0.02118826707204183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,7168,0.058543999989827476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,1536,0.017656532923380534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,1536,0.02449280023574829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,1024,0.012967466314633688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,1024,0.021513599157333373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,6144,0.048798934618632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,768,0.010843732953071594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,768,0.02018453280131022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,4096,0.03539733489354451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,5120,0.04306559960047404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,512,0.00840106705824534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,512,0.018142932653427125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,256,0.006293333570162455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,256,0.017493333419164023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,3584,0.031981867551803586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,256,0.008666666348775227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,128,0.005366399884223938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,128,0.008008533219496409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,128,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,3072,0.027229867378870648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,64,0.0050794666012128195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,32,0.005142400165398916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,64,0.01760960022608439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,32,0.017537067333857216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,65536,0.4023338635762532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,65536,0.532209078470866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,2560,0.024984532594680788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,51200,0.419538148244222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,51200,0.31947946548461914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,1536,0.017646932601928712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,1024,0.01511679987112681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,16384,0.13695893287658692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,16384,0.10629653135935466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,12288,0.10442453225453693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,12288,0.094650665918986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,10240,0.08756266434987386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,10240,0.07679893175760905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,10240,0.07138773600260416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,8192,0.07074027061462403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,8192,0.06113066673278809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,768,0.01234879990418752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,7168,0.062193067868550625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,7168,0.05436906814575195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,6144,0.05170773267745972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,6144,0.04891200065612793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,512,0.010326400399208069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,5120,0.04456640084584554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,5120,0.043112532297770186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,65536,0.49886185328165694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,4096,0.03692906697591146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,4096,0.04042239983876546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,16384,0.11559786796569824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,3584,0.03226666649182637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,51200,0.4032138824462891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,3584,0.03425706624984741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,3072,0.028058665990829467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,12288,0.09367787043253581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,3072,0.031395200888315836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,2560,0.024154667059580484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,2560,0.028497066100438433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,8192,0.05752213398615519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,2048,0.0200437327226003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,7168,0.051140268643697105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,2048,0.027217066287994383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,1536,0.016315733393033348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,1536,0.024530132611592613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,6144,0.045748265584309895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,1024,0.01225279966990153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,1024,0.02178666591644287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,5120,0.037590400377909346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,768,0.010404266913731893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,768,0.01991573373476664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,2048,0.019361066818237304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,512,0.00848746697107951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,4096,0.03184426625569661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,512,0.018669867515563966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,3584,0.028742400805155437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,256,0.0061749334136645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,3072,0.02490239938100179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,256,0.01776426633199056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,128,0.0052149335543314615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,256,0.008573866883913676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,2560,0.022334933280944824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,128,0.017010132471720375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,64,0.004632533093293508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,64,0.017167999347050985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,32,0.01717546582221985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,1536,0.016356266538302102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,65536,0.44995838801066085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,1024,0.013969066739082336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,65536,0.37161601384480797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,768,0.011582932869593303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,51200,0.2848959922790527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,32,0.004957866668701172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,16384,0.11585066318511963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,16384,0.09434560139973959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,12288,0.08728319803873698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,12288,0.0748095989227295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,10240,0.07454293568929037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,10240,0.06625493367513022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,512,0.009898666540781658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,51200,0.350767993927002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,8192,0.05837546586990357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,8192,0.05540800094604492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,7168,0.051453868548075356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,7168,0.050513064861297606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,128,0.007980800171693166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,6144,0.044828800360361735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,6144,0.04562559922536214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,12288,0.08611093362172445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,5120,0.03847786585489909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,5120,0.040354132652282715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,65536,0.4739839871724446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,4096,0.03240106701850891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,4096,0.03607893387476603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,16384,0.11208000183105468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,3584,0.028357332944869994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,3584,0.0325109342734019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,51200,0.36802988052368163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,3072,0.024996266762415568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,6144,0.04101119836171468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,10240,0.06909759839375815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,2560,0.02169706622759501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,3072,0.02972373366355896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,2560,0.028010666370391846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,2048,0.01812053322792053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,2048,0.025758934020996094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,7168,0.044777599970499675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,1536,0.014828800161679586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,1536,0.023745065927505492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,8192,0.051420799891153966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,1024,0.011598933736483257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,1024,0.02121493419011434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,3072,0.02250880002975464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,768,0.010408533612887065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,768,0.02049493392308553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,1536,0.014847999811172486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,512,0.00831573357184728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,512,0.01900906761487325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,4096,0.02918720046679179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,256,0.006397866706053417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,256,0.017832533518473307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,2560,0.0207914670308431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,128,0.005134933193524678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,128,0.01676373283068339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,5120,0.03410880168279012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,64,0.0046517332394917805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,3584,0.026477867364883424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,64,0.016684800386428833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,32,0.0048991998036702475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,32,0.016759467124938966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,1024,0.01281599998474121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,2048,0.017731199661890663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,65536,0.3578101476033529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,65536,0.30923840204874675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,51200,0.2865205446879069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,51200,0.24815893173217773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,16384,0.09167040189107259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,16384,0.08359999656677246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,12288,0.06851200262705484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,12288,0.06612266699473063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,128,0.0072618668278058365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,10240,0.05705386797587077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,256,0.007879466811815897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,10240,0.057145599524180093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,8192,0.046199464797973634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,512,0.008996267120043437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,8192,0.04890880187352498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,7168,0.04088533322016398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,7168,0.04452799956003825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,768,0.010696533322334289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,6144,0.03509866793950399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,6144,0.040010666847229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,51200,0.24907093048095702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,65536,0.3075765291849772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,5120,0.02986239989598592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,5120,0.03499093453089396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,12288,0.05701653162638346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,4096,0.024877866109212242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,4096,0.030829866727193195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,16384,0.0720970630645752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,3584,0.022317866484324135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,3584,0.029036800066630047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,3072,0.019167999426523842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,3072,0.027151999870936076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,10240,0.048554666837056476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,2560,0.016532267133394875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,2560,0.025613866249720257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,7168,0.03375039895375569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,2048,0.013959466417630514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,2048,0.02360639969507853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,8192,0.0371829350789388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,1536,0.011754666765530903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,1536,0.021811199188232423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,6144,0.031229867537816362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,1024,0.009139200051625569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,1024,0.019564799467722573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,5120,0.025603200991948443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,768,0.007605333129564922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,768,0.01815999945004781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,4096,0.022111999988555908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,512,0.006274133423964183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,512,0.017707733313242595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,2560,0.016025599837303162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,256,0.005126399795214335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,256,0.016740266482035318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,3584,0.02037013371785482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,128,0.004509866734345754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,128,0.016247466206550598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,2048,0.014306132992108664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,64,0.0039989332358042395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,64,0.016116266449292503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,32,0.004163199911514918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,32,0.016238933801651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,3072,0.01768640081087748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,65536,0.3095775922139486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,65536,0.28489386240641273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,51200,0.24267093340555826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,51200,0.22878185907999674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,1536,0.012779733538627625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,16384,0.07768320242563884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,16384,0.07577280203501383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,768,0.009133866429328919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,12288,0.05938453276952108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,128,0.007233066856861115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,1024,0.01060693363348643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,12288,0.061572265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,10240,0.05464213291803995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,10240,0.05370879968007406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,8192,0.04036266803741455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,8192,0.0458837350209554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,256,0.007593599955240886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,7168,0.03597653309504191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,512,0.008405333757400513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,7168,0.0418176015218099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,6144,0.03135679960250855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,6144,0.03784533341725667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,65536,0.2721226692199707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,5120,0.027765333652496338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,5120,0.032756266991297404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,12288,0.05406506856282552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,4096,0.022346667448679605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,4096,0.029497599601745604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,16384,0.06877439816792806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,3584,0.019332265853881835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,3584,0.027089067300160724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,6144,0.029289599259694415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,3072,0.017147733767827352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,3072,0.0253983994325002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,51200,0.21944746971130372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,2560,0.014932266871134438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,2560,0.02390399972597758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,10240,0.0445248007774353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,2048,0.012546133001645407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,2048,0.022221867243448892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,7168,0.03295253316561381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,1536,0.010651733477910359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,1536,0.019809067249298096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,8192,0.03736960093180339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,1024,0.00841813286145528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,1024,0.01834453344345093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,3072,0.01738133430480957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,768,0.006971733272075653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,768,0.018175999323527016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,4096,0.021673599878946938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,512,0.0058794667323430385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,512,0.01725546717643738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,1536,0.011904000242551168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,256,0.004901333153247834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,256,0.016264533003171287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,5120,0.025483733415603636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,128,0.004253866771856943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,128,0.01575146714846293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,2560,0.015588266650835672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,3584,0.019833600521087645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,64,0.0039007999002933502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,64,0.015944533546765647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,32,0.004074666649103165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,32,0.01594239970048269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,65536,0.2725045204162598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,65536,0.29090452194213867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,2048,0.013529599706331889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,51200,0.21683947245279947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,1024,0.009569066762924194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,51200,0.2155893325805664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,16384,0.0686517318089803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,16384,0.07191039721171061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,12288,0.05291733344395956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,12288,0.05838506619135538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,256,0.006687999765078227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,128,0.006198399762312571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,10240,0.04578773180643718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,10240,0.05158079862594604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,8192,0.035860268274943034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,8192,0.043066668510437014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,512,0.007325866818428039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,7168,0.032458666960398355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,7168,0.03864853382110596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,768,0.008221866687138875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,6144,0.02768320043881734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,6144,0.03510719935099284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,65536,0.25746347109476725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,5120,0.024565333127975465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,5120,0.03146666685740153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,16384,0.0642687996228536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,4096,0.019189333915710448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,4096,0.02754559914271037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,12288,0.04738239844640096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,3584,0.017117865880330405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,51200,0.20525867144266763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,3584,0.026794666051864625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,3072,0.015440000096956888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,3072,0.02437333265940348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,10240,0.04539200067520142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,2560,0.013507200280825296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,2560,0.022992000977198283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,6144,0.026266666253407796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,2048,0.011428266763687134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,2048,0.02143893241882324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,7168,0.030219733715057373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,1536,0.009796266754468281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,1536,0.01989013353983561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,8192,0.033411200841267905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,1024,0.007347199817498524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,1024,0.018245333433151247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,3072,0.015713066856066386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,768,0.006365866462389629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,768,0.017883733908335368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,4096,0.019381332397460937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,512,0.005481599768002828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,512,0.017084799210230508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,5120,0.02314773400624593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,256,0.00451200008392334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,256,0.016280532876650492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,3584,0.018217599391937254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,128,0.004018133382002512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,128,0.015701333681742348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,1536,0.010875733693440755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,64,0.0037109332780043284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,64,0.01584106683731079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,32,0.0038624001046021783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,32,0.015947733322779337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,65536,0.23595199584960938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,2560,0.01455893317858378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,65536,0.2654400030771891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,51200,0.18898666699727376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,51200,0.20590933163960776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,1024,0.00902826686700185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,16384,0.05922133525212606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,2048,0.012757333119710288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,16384,0.0674944003423055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,12288,0.053990399837493895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,12288,0.058619733651479086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,256,0.006515199939409892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,10240,0.04742613236109416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,128,0.0061034664511680605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,10240,0.05309653282165527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,512,0.007167999943097432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,8192,0.0407968004544576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,768,0.007850666840871174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,7168,0.03043839931488037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,7168,0.03555733362833659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,8192,0.03216106692949931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,6144,0.028769065936406452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,6144,0.03445546627044678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,65536,0.24668800036112465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,5120,0.024604799350102742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,5120,0.031007999181747438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,16384,0.05868800083796183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,4096,0.016591999928156534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,4096,0.027416533231735228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,51200,0.18306026458740235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,12288,0.04440639813741048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,3584,0.015301332871119181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,3584,0.025193599859873454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,10240,0.03998399972915649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,3072,0.015387733777364096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,3072,0.024578134218851723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,6144,0.024050132433573405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,2560,0.013242666920026144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,7168,0.02717333237330119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,2048,0.010705066720644633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,2048,0.021227733294169108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,8192,0.03136533300081889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,1536,0.009249066313107807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,1536,0.019658666849136353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,2560,0.021886932849884033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,1024,0.006720000008742015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,1024,0.01818986733754476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,4096,0.018013866742451985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,768,0.006020266811052958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,768,0.017414400974909462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,5120,0.021466666460037233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,512,0.005060266455014547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,512,0.01699519952138265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,3072,0.014842666188875833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,256,0.0043029333154360454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,256,0.016212266683578492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,3584,0.01700906753540039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,128,0.003945599993069967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,128,0.015518933534622192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,1536,0.009914666414260864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,64,0.0036608000596364343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,64,0.015594666202863058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,32,0.0037280000746250153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,32,0.01563093364238739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,65536,0.18160427411397298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,2560,0.01350826621055603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,65536,0.23012053171793617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,51200,0.14717226028442382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,51200,0.18194986979166666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,1024,0.00853013296922048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,16384,0.053191467126210534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,2048,0.011678933103879293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,16384,0.06611946821212769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,12288,0.050179199377695716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,12288,0.05760533412297567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,512,0.006805333495140076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,10240,0.042277332146962485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,10240,0.049403735001881915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,256,0.006126933296521505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,8192,0.0325055996576945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,8192,0.038260265191396074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,768,0.007451733450094859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,7168,0.0285589337348938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,7168,0.03610560099283854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,128,0.005702400207519531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,6144,0.026405332485834758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,6144,0.032979200283686325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,65536,0.24025707244873046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,5120,0.022094933191935222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,5120,0.02994453310966492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,51200,0.19356053670247395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,16384,0.05587413311004639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,4096,0.018268799781799315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,12288,0.046238934993743895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,3584,0.01622293293476105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,3584,0.02397226691246033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,10240,0.040040532747904464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,3072,0.014417066176732381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,3072,0.022895999749501548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,4096,0.0268832008043925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,8192,0.030933332443237305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,2560,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,2560,0.021107200781504312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,2048,0.010919466614723206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,2048,0.019500799973805747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,7168,0.02695253292719523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,1536,0.009204266468683879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,1536,0.019127466281255088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,6144,0.024439465999603272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,1024,0.006629333396752675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,1024,0.017861332496007284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,5120,0.02116159995396932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,768,0.005736533304055532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,768,0.017333332697550455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,4096,0.017526400089263917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,512,0.004880000154177348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,512,0.016242133577664693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,2048,0.010752000411351522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,3072,0.014521599809328715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,256,0.00413226659099261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,256,0.015979733069737753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,3584,0.016820265849431356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,128,0.01530346671740214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,1024,0.007993599772453308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,64,0.0034656000634034475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,64,0.015388799707094827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,32,0.003554133325815201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,32,0.015518933534622192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,65536,0.14176534016927084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,2560,0.012703999876976013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,65536,0.2067018667856852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,128,0.0036906667053699495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,51200,0.11355199813842773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,51200,0.1642570654551188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,1536,0.009427199761072796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,16384,0.04108373324076335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,16384,0.06488746802012126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,768,0.0072629332542419435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,12288,0.042599467436472575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,12288,0.05283519824345907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,512,0.006727466483910878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,10240,0.038072534402211505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,10240,0.04559146563212077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,256,0.0063498665889104204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,128,0.005884799857934316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,8192,0.027802666028340656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,8192,0.036697598298390706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,7168,0.025434666872024538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,7168,0.033532798290252686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,6144,0.022037333250045775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,6144,0.0309663991133372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,65536,0.21333759625752768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,5120,0.01917866667111715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,5120,0.02778880000114441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,51200,0.1690293312072754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,16384,0.049030399322509764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,4096,0.015532799561818442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,4096,0.02528426647186279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,3584,0.013704533378283182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,3584,0.02363306681315104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,12288,0.03726933399836223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,3072,0.012906666596730551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,3072,0.02225066622098287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,10240,0.03310826619466146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,2560,0.01099733312924703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,2560,0.02164586583773295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,7168,0.02326293389002482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,2048,0.009461333354314167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,8192,0.026190932591756182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,6144,0.020768000682195028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,2048,0.019469867149988808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,1536,0.007282133400440216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,1536,0.01853013237317403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,1024,0.0057770664493242896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,1024,0.017139200369517008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,5120,0.01798293391863505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,768,0.005025066435337067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,768,0.01639573375384013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,3584,0.014628266294797262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,512,0.004266666869322458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,512,0.016204800208409628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,3072,0.012244266271591187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,256,0.0037791999677817024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,4096,0.01532373329003652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,256,0.015383467078208923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,128,0.0034048000971476236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,128,0.015001599987347921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,1536,0.00858026643594106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,2560,0.011559466520945232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,64,0.0032127998769283296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,32,0.0033290666838486993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,64,0.014851199587186179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,32,0.015025066335995993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,65536,0.09670613606770834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,65536,0.18429759343465169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,51200,0.07782933712005616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,1024,0.007512533167997997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,51200,0.14902186393737793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,16384,0.03110719919204712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,16384,0.05870933135350546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,2048,0.009756799538930256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,12288,0.029494400819142657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,12288,0.04610133171081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,768,0.006614399949709575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,10240,0.027918932835261027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,10240,0.0401525338490804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,512,0.006038400034109751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,8192,0.021062399943669638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,256,0.005602133274078369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,8192,0.03350720008214315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,7168,0.017485866943995156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,7168,0.031242666641871135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,128,0.005294933418432872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,6144,0.015377066532770791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,6144,0.028520532449086505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,65536,0.20380053520202637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,5120,0.015381333231925965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,5120,0.025639466444651288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,51200,0.15891733169555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,4096,0.011825066804885865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,4096,0.022296533981959025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,16384,0.041774932543436685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,3584,0.010909866293271382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,12288,0.03347200155258179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,3584,0.02109760046005249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,3072,0.009037866195042928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,3072,0.02044586737950643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,10240,0.029064534107844035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,2560,0.007674666742483775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,2560,0.01935360034306844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,8192,0.022410666942596434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,2048,0.0072170664866765336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,7168,0.020436267058054604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,2048,0.019203199942906698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,6144,0.018744534254074095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,1536,0.005838933090368906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,1536,0.017335466543833413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,5120,0.016074666380882265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,1024,0.0049685334165891016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,1024,0.0164682666460673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,4096,0.01383039951324463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,768,0.004461866617202759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,768,0.016311466693878174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,3072,0.011277866363525391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,512,0.003978666663169861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,512,0.016214399536450704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,3584,0.01309333344300588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,256,0.0035466666022936503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,256,0.015476266543070475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,128,0.0033215999603271483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,128,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,2048,0.008876799543698629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,64,0.0030826665461063385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,64,0.015277866522471109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,32,0.003202133377393087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,32,0.015269333124160766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,65536,0.07731733322143555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,2560,0.01032533347606659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,65536,0.1754101276397705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,51200,0.06203946669896444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,51200,0.1411253293355306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,1024,0.0070783997575442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,512,0.006018133461475372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,16384,0.02611946662267049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,16384,0.05271999835968018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,1536,0.008342400193214417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,768,0.006312533219655354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,12288,0.02433919906616211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,12288,0.039738667011260984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,10240,0.02219946583112081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,256,0.00550186683734258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,10240,0.03519573211669922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,8192,0.016695467631022136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,8192,0.030555733044942218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,7168,0.01504853367805481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,128,0.005260799825191498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,7168,0.02866986592610677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,6144,0.013390933473904928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,6144,0.0264629324277242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,65536,0.17202986081441243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,5120,0.013431466619173684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,5120,0.023938133319218954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,51200,0.13426559766133625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,4096,0.010910933216412861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,4096,0.022740266720453896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,12288,0.03300586740175883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,3584,0.010005333026250203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,3584,0.021681066354115805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,16384,0.04431573152542114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,3072,0.008590933680534363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,6144,0.018950400749842326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,3072,0.020060799519220986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,2560,0.008309333523114523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,2560,0.019740800062815346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,7168,0.021310933430989585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,2048,0.00689386675755183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,2048,0.018739199638366698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,8192,0.023859200874964397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,1536,0.006040533383687338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,1536,0.017875200510025023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,10240,0.0286954661210378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,1024,0.004700799783070883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,1024,0.01690666675567627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,3072,0.010959999759991963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,768,0.004166399935881296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,4096,0.013674666484196981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,768,0.01627306640148163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,5120,0.01606826682885488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,512,0.003730133424202601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,512,0.016344533363978068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,256,0.003382399926582972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,256,0.015334399541219077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,3584,0.013063466548919678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,128,0.0031360000371932983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,128,0.015094400445620219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,2560,0.010227200388908387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,1536,0.008131200075149536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,64,0.002974933385848999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,32,0.003101866692304611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,64,0.015406933426856995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,32,0.014890666802724203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,65536,0.058169599374135336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,65536,0.166648530960083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,51200,0.04867306550343831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,2048,0.00897279977798462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,51200,0.13351999918619792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,16384,0.019986132780710854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,16384,0.04814186493555705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,1024,0.007152000069618225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,12288,0.01907306710879008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,12288,0.039469865957895915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,512,0.005938133100668589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,256,0.005383466680844625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,10240,0.018103466431299845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,128,0.0050687998533248905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,768,0.006374399860699971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,10240,0.03531413475672404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,8192,0.012905599673589072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,8192,0.029131732384363812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,7168,0.01225386659304301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,7168,0.027955200274785357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,65536,0.1603498617808024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,6144,0.01406613290309906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,6144,0.026042666037877398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,51200,0.13102293014526367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,5120,0.012583466370900473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,16384,0.038942933082580566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,5120,0.023884799083073935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,4096,0.009187199672063192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,4096,0.022578134139378866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,12288,0.029470932483673096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,3584,0.008706133564313252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,3584,0.021050665775934854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,3072,0.007979733248551685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,3072,0.020837332804997763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,10240,0.025675733884175617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,2560,0.0073749333620071415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,8192,0.02107200026512146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,2560,0.01917120019594828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,2048,0.0066655998428662615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,7168,0.019050665696461997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,2048,0.018243199586868285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,1536,0.005799466868241628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,1536,0.01730133295059204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,6144,0.01691306630770365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,1024,0.004829866687456766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,1024,0.016501333316167197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,5120,0.014599466323852539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,768,0.004277333120505015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,768,0.01606186628341675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,4096,0.012386133273442585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,512,0.003914666672547659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,512,0.015800533692042033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,3072,0.010331733028093974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,256,0.0033386667569478357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,256,0.015020799636840821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,3584,0.01223360002040863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,128,0.003123199939727783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,128,0.014968533317248026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,2048,0.008380800485610962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,64,0.0029834667841593427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,64,0.014683733383814493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,32,0.0029418667157491045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,32,0.014782933394114175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,65536,0.042481064796447754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,2560,0.009829333424568177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,65536,0.15584425926208495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,51200,0.03521173397699992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,65536,0.14719467163085936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,1024,0.006900266806284587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,51200,0.1250933329264323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,51200,0.11599360307057698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,16384,0.012903466820716858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,12288,0.01120746632417043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,16384,0.03785813252131144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,16384,0.040402134259541825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,10240,0.010431999961535137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,12288,0.032585599025090534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,10240,0.02999146580696106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,8192,0.008200533191363017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,768,0.006041599810123444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,8192,0.019886932770411172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,7168,0.010823466380437215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,7168,0.02558186650276184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,512,0.0057888001203536986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,6144,0.008351999521255492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,7168,0.018083200852076212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,6144,0.02514773408571879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,5120,0.00862613320350647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,5120,0.023490132888158162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,1536,0.008164266745249431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,4096,0.007916800181070964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,4096,0.021461333831151327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,256,0.005236266553401947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,3584,0.007784533500671387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,128,0.004970666766166687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,3584,0.011925333738327026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,3072,0.00745066652695338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,8192,0.027662932872772217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,3072,0.019595734278361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,2560,0.006491733094056447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,2560,0.019273600975672402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,3584,0.02020053267478943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,2048,0.005730133255322774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,2048,0.01812480092048645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,12288,0.028974932432174683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,10240,0.02439360022544861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,1536,0.004995200037956238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,1536,0.016979199647903443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,1536,0.007783466577529907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,1024,0.004331733286380768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,1024,0.0067456002036730455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,1024,0.016475733121236166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,768,0.003927466770013174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,768,0.016028799613316855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,768,0.005967999994754791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,512,0.0035616000493367515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,6144,0.016006400187810264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,512,0.01569066643714905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,256,0.0031829332311948144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,256,0.014913066228230795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,5120,0.014146133263905843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,128,0.003035733352104823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,128,0.014801067113876343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,128,0.004819199939568837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,64,0.002856533229351044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,64,0.01546986699104309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,32,0.0029205332199732465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,32,0.014599466323852539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,4096,0.012411733468373615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,65536,0.03481386502583821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,65536,0.13627947171529134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,65536,0.1515391985575358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,51200,0.02831466595331828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,51200,0.1206986665725708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,16384,0.012495999534924824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,3072,0.010251733660697936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,16384,0.037361065546671554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,12288,0.009081600109736125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,2560,0.009585066636403402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,12288,0.03206293384234111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,10240,0.009692800045013428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,10240,0.028588799635569255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,256,0.0050240000089009605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,8192,0.007778133451938629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,8192,0.02704746723175049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,8192,0.019935999313990274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,7168,0.007650133470694225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,7168,0.024727465709050496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,2048,0.008374399940172831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,6144,0.008462933699289958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,6144,0.023550933599472045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,6144,0.01609599987665812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,5120,0.007770666480064392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,5120,0.022987733284632363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,4096,0.0071263998746871945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,4096,0.02120959957440694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,512,0.005559466779232025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,3584,0.006741333504517872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,3584,0.019964800278345744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,51200,0.10802773634592693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,3072,0.006403199831644694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,3072,0.020102399587631225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,16384,0.03617279926935832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,2560,0.006251733501752217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,12288,0.02789013385772705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,10240,0.024065067370732628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,2048,0.00556160012880961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,2048,0.018310399850209554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,7168,0.01804373264312744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,1536,0.004814933240413666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,1536,0.01691626707712809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,1536,0.0077461332082748415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,1024,0.004181333382924398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,1024,0.01621333360671997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,2560,0.01869759956995646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,768,0.0038783999780813852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,5120,0.014099199573198953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,768,0.016082132856051125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,512,0.003521066655715307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,512,0.01564479966958364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,4096,0.012113066514333089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,512,0.005677866439024607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,256,0.0032032000521818793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,256,0.015038933356602988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,256,0.005156266689300537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,128,0.002961066613594691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,128,0.014707199732462563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,64,0.0028970666229724885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,3584,0.011758933464686077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,64,0.015000533064206442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,32,0.0029706666866938275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,32,0.01474346617857615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,65536,0.03159679969151814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,51200,0.025784534215927125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,65536,0.14759146372477214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,16384,0.007321600119272869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,51200,0.1175434668858846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,12288,0.00769706666469574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,16384,0.036101333300272626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,12288,0.029787733157475786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,10240,0.007272533575693766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,8192,0.006694399813810985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,10240,0.02790186603864034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,7168,0.00647573322057724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,8192,0.025312000513076784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,6144,0.006211199859778086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,7168,0.024270933866500855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,5120,0.0065184002121289565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,6144,0.023242666323979696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,4096,0.006202666461467743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,5120,0.02249173323313395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,3072,0.010126933455467224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,3584,0.006567466755708058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,3584,0.0202346662680308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,3072,0.006212266782919565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,3072,0.019000534216562906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,2560,0.006120533247788747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,2560,0.018476800123850504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,2048,0.005422933399677277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,2048,0.017963733275731406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,2048,0.00819413314263026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,1536,0.004774400095144907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,1024,0.00405973345041275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,1536,0.01731946667035421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,1024,0.01636799971262614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,768,0.003789866715669632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,512,0.003389866650104523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,768,0.015966932972272238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,256,0.003102933367093404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,512,0.01586560010910034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,128,0.002868266652027766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,128,0.01477226714293162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,64,0.0027456000447273255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,64,0.014587733149528503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,32,0.0027818667391935987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,1024,0.006698666512966156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,32,0.014566399653752646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,768,0.00595413347085317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,65536,0.02929813265800476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,51200,0.024405332406361897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,16384,0.007451733450094859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,4096,0.021329067150751748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,51200,0.11703466574350993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,12288,0.006742399930953979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,16384,0.03501760164896647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,10240,0.00653653343518575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,12288,0.029257599512736005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,8192,0.006436266501744588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,10240,0.027607466777165728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,2560,0.009476266304651896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,7168,0.00631573349237442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,8192,0.025460267066955568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,6144,0.005998933315277099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,7168,0.02472320000330607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,5120,0.006386133531729381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,6144,0.02334400018056234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,4096,0.005994666616121928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,3584,0.006468266745408376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,5120,0.02288533250490824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,4096,0.020776534080505372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,3072,0.006213333209355672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,2560,0.006105599800745646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,3584,0.020334933201471964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,3072,0.01964799960454305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,2048,0.005386666456858317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,2560,0.018463999032974243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,1536,0.004722133278846741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,2048,0.017588265736897788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,1024,0.004175999760627746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,1536,0.016992000738779704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,768,0.0037791999677817024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,1024,0.01634773313999176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,512,0.0033482665816942847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,768,0.016034133235613503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,256,0.0030602666238943735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,512,0.015421866377194723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,128,0.002930133293072383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,256,0.014712533354759217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,128,0.004890666902065277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,64,0.0027690666417280836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,128,0.014819199840227762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,32,0.002713600049416224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,64,0.014644267161687216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,32,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,256,0.014847999811172486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,65536,0.14812053044637044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,16384,0.44701547622680665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,16384,0.8214303970336914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,16384,1.0537738800048828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,12288,0.6318080266316731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,12288,0.34711147944132487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,10240,0.2935818672180176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,10240,0.5183519999186198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,8192,0.4176063855489095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,12288,0.792195192972819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,51200,1.3534912109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,10240,0.6909013112386068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,8192,0.24176106452941895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,7168,0.3710890769958496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,7168,0.21732692718505858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,6144,0.19163947105407714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,6144,0.3237514813741048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,5120,0.2701141357421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,5120,0.16560959815979004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,51200,2.6558270772298176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,4096,0.22297066052754722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,3584,0.20102400779724122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,3584,0.12864747047424316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,3584,0.2577119986216227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,51200,3.6043861389160154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,3072,0.17333332697550458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,3072,0.12424426873524982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,4096,0.13842239379882812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,2560,0.1475722630818685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,2560,0.09741973082224528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,2560,0.18248106638590494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,2048,0.12157119909922283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,2048,0.08234240214029948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,8192,0.5886112213134765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,1536,0.10341973304748535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,7168,0.5213216145833333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,1536,0.06866453488667806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,5120,0.3697888056437174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,1024,0.06728533109029135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,1536,0.11149973074595135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,768,0.05496960083643595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,1024,0.055693864822387695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,4096,0.28956801096598306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,512,0.04700053135553996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,512,0.04692266782124837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,512,0.049958399931589764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,256,0.03006826639175415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,6144,0.4449909210205078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,256,0.04283519983291626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,256,0.036660265922546384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,128,0.025094399849573772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,128,0.0320362667242686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,64,0.017537067333857216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,65536,32,0.01776426633199056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,64,0.030727465947469074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,1024,0.0800661325454712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,3072,0.22146132787068687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,32,0.03231786688168843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,65536,768,0.04927359819412232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,2048,0.14964693387349445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,768,0.06187946796417236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,65536,1.6827968597412108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,65536,128,0.03840320110321045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,51200,2.7618250528971355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,16384,0.728769048055013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,16384,0.4487765312194824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,51200,1.2751487731933593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,65536,3.202643076578776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,12288,0.3146176020304362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,16384,0.8423946380615235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,12288,0.6347765604654948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,10240,0.46982825597127276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,10240,0.2901280085245768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,8192,0.3652053197224935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,8192,0.2239744027455648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,7168,0.19602133433024088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,7168,0.36897811889648435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,8192,0.48120854695638016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,6144,0.1682794729868571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,6144,0.2827989260355631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,5120,0.1449077288309733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,5120,0.23696746826171874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,4096,0.1914517402648926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,5120,0.2956000010172526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,4096,0.13115306695302326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,3584,0.17075306574503582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,3584,0.11025599638621013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,12288,0.6459648132324218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,51200,2.87891845703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,65536,3.832912953694662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,3072,0.16153173446655272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,10240,0.5684117635091146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,3072,0.1092362642288208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,2560,0.1274517297744751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,2560,0.08864959875742594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,2048,0.10316373507181804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,2048,0.07250986893971761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,7168,0.42054080963134766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,1536,0.08144319852193196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,1536,0.06076586643854777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,1024,0.05722879966100057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,1024,0.049139201641082764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,6144,0.35585066477457683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,768,0.05224639972050985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,768,0.043644801775614424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,4096,0.23494292894999186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,512,0.03906773328781128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,512,0.037562668323516846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,3584,0.20863893826802574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,256,0.02458560069402059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,256,0.031062400341033934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,3072,0.17718186378479003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,2560,0.14864959716796874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,128,0.01830613414446513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,2048,0.12171839872996013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,128,0.02820906639099121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,1536,0.09143786430358887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,64,0.02806719938913981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,32,0.015086932977040609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,51200,32,0.028756266832351683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,1024,0.0673845370610555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,768,0.051151998837788905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,65536,0.4752981185913086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,65536,0.805290667215983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,51200,64,0.01456106702486674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,512,0.039316264788309734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,51200,0.6327477137247721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,16384,0.2057759920756022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,51200,0.37888959248860676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,256,0.028823467095692952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,16384,0.15470080375671386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,12288,0.16780799229939777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,12288,0.10696213245391846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,16384,0.26472427050272623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,12288,0.20143680572509765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,10240,0.13313600222269695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,10240,0.09189759890238444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,51200,0.8286720275878906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,8192,0.1087114651997884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,8192,0.07831892967224122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,8192,0.15568854014078776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,7168,0.07088639736175537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,7168,0.11096853415171307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,6144,0.08329599698384603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,6144,0.06370773315429687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,5120,0.07012480099995931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,6144,0.12035306294759114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,5120,0.05544960101445516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,5120,0.0991424004236857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,4096,0.0583402673403422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,4096,0.04969066778818766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,3584,0.05249493519465128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,4096,0.07893653710683188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,3072,0.04591253201166789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,51200,128,0.024119466543197632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,3072,0.040703999996185306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,2560,0.039342931906382245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,3072,0.06202239990234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,2560,0.036711466312408444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,2048,0.03349973360697429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,2048,0.03228480021158854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,3584,0.04568106730779012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,1536,0.026862933238347368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,1536,0.027924267450968425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,65536,1.080185572306315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,1024,0.01882773240407308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,1024,0.02429973284403483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,768,0.014888532956441245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,768,0.022664533058802287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,10240,0.17800960540771485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,512,0.011572266618410747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,512,0.021347200870513915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,512,0.01685973405838013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,7168,0.13626133600870768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,256,0.008770133058230083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,256,0.018090667327245076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,128,0.006664533416430156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,128,0.017755732933680216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,3584,0.07055466969807943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,64,0.00565226674079895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,1024,0.02605653405189514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,64,0.01826559901237488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,16384,32,0.00622506688038508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,16384,32,0.01847040057182312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,2560,0.05157226721445719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,2048,0.042320001125335696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,65536,0.3885301272074381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,65536,0.6115968068440755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,51200,0.47315521240234376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,16384,0.16755092938741048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,1536,0.03162879943847656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,51200,0.3047712008158366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,16384,0.10984640121459961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,12288,0.12172373135884602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,12288,0.08727040290832519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,768,0.020111999909083047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,10240,0.09980053106943766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,10240,0.0744704008102417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,8192,0.08010559876759847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,8192,0.0630186676979065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,256,0.013879467050234476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,7168,0.0721407969792684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,16384,128,0.012453333536783854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,7168,0.057284267743428555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,6144,0.063155198097229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,6144,0.051913599173227944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,12288,0.17917332649230958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,5120,0.054059731960296634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,5120,0.045960533618927005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,65536,0.9127701441446939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,16384,0.22895359992980957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,4096,0.050385065873463954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,4096,0.04038613239924113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,3584,0.03979413509368897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,6144,0.10094613234202068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,3584,0.03791679938634236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,3072,0.035240534941355386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,3072,0.0348042647043864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,10240,0.16394346555074055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,2560,0.030935466289520264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,2560,0.031143466631571453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,51200,0.706388282775879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,2048,0.028038400411605834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,7168,0.12086719671885174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,2048,0.027003733317057292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,1536,0.02022506594657898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,8192,0.13940906524658203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,1536,0.024693334102630617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,1024,0.01420799990495046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,1024,0.022779732942581177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,3072,0.053345068295796716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,768,0.011843199531237285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,768,0.021076265970865884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,1536,0.02876586715380351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,512,0.009546666344006857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,4096,0.07137706279754638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,512,0.019562667608261107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,256,0.006761600077152252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,256,0.01742186745007833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,2560,0.04625920057296753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,128,0.005353599786758423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,5120,0.08737706343332927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,128,0.01689493258794149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,64,0.00477866679430008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,64,0.019751467307408652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,12288,32,0.0077450667818387345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,12288,32,0.020129066705703736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,3584,0.060977065563201906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,2048,0.038038400808970134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,65536,0.5754592259724934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,65536,0.35719359715779625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,51200,0.2815615971883138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,51200,0.4481696128845215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,16384,0.1651253382364909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,16384,0.10518399874369304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,51200,0.5682645161946615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,12288,0.11264426708221435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,12288,0.0818826675415039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,12288,0.13860799471537272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,10240,0.09508480230967203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,10240,0.07145706812540689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,10240,0.12377493381500244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,8192,0.07520533402760823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,8192,0.060828800996144614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,1024,0.020619734128316244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,7168,0.06847146352132162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,7168,0.05561173359553019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,7168,0.09638400077819824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,6144,0.06007039944330851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,6144,0.049932801723480226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,256,0.010789333780606588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,128,0.009597866733868917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,5120,0.05034666856129964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,5120,0.04528426726659139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,4096,0.04121493498484294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,4096,0.03861440022786458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,512,0.01428053379058838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,12288,768,0.017874133586883546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,3584,0.037656533718109134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,3584,0.03667519887288411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,3072,0.03323093255360921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,3072,0.03327893416086833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,3072,0.042080001036326094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,2560,0.028540800015131634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,2560,0.029715200265248615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,65536,0.7292800267537435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,2048,0.026366933186848955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,2048,0.027229867378870648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,16384,0.1834837277730306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,1536,0.01885226567586263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,1536,0.02378773291905721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,1536,0.023228800296783446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,1024,0.012730666995048523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,1024,0.02142080068588257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,1024,0.01725333333015442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,768,0.011236266295115153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,768,0.020461867252985634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,6144,0.08224106629689534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,512,0.008669867118199667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,512,0.018130133549372353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,8192,0.10439893404642742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,256,0.005982933441797892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,256,0.017757866779963175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,4096,0.05592319965362549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,128,0.005341866612434387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,128,0.01676586667696635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,2560,0.03564266761144002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,64,0.004622933268547058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,64,0.01693333387374878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,10240,32,0.004762666424115499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,10240,32,0.016798933347066246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,5120,0.06837226549784342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,65536,0.2720565478006999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,65536,0.42241493860880536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,51200,0.3357664108276367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,51200,0.2199317296346029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,3584,0.048640000820159915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,16384,0.105293869972229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,16384,0.08267947038014731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,512,0.012049067020416259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,12288,0.08192000389099122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,12288,0.06733866532643637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,128,0.008385066191355388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,256,0.009486933549245197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,10240,0.06899733543395996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,10240,0.05799359877904257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,768,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,8192,0.05559680064519247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,8192,0.04981866677602132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,10240,2048,0.030892799297968548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,7168,0.04981653292973836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,7168,0.045032533009847005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,6144,0.04412906567255656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,6144,0.041900801658630374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,65536,0.6327264149983723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,5120,0.037732267379760744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,5120,0.037356801827748615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,16384,0.1664021333058675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,4096,0.031048534313837688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,4096,0.033469867706298825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,12288,0.1280789295832316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,3584,0.028088533878326417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,3584,0.029553065697352093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,51200,0.49038400650024416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,3072,0.027806933720906573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,3072,0.027703466018040974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,10240,0.11333973407745361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,2560,0.021337600549062093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,2560,0.025260800123214723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,8192,0.09866773287455241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,2048,0.017785600821177163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,2048,0.023264000813166298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,6144,0.07171946366628011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,7168,0.08559359709421793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,1536,0.013527466853459676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,1536,0.021896533171335855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,1024,0.010275200009346008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,1024,0.020240000883738198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,3072,0.03923199971516927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,768,0.008854400118192036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,768,0.019543466965357463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,5120,0.06254719893137614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,4096,0.05098453362782797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,512,0.007550933460394542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,512,0.01742186745007833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,256,0.0053162669142087305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,256,0.017617066701253258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,3584,0.045135998725891115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,128,0.00461760014295578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,128,0.016169599692026772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,2560,0.03346133232116699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,64,0.004145066688458125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,64,0.01625279982884725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,8192,32,0.0044608001907666525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,8192,32,0.0165173331896464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,65536,0.4124330520629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,1536,0.021223467588424683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,65536,0.30099093119303383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,2048,0.028344533840815228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,51200,0.32981974283854165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,51200,0.24509760538736977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,1024,0.01614400049050649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,16384,0.1516991933186849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,16384,0.08857173124949137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,12288,0.07595199743906657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,12288,0.07167147000630697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,256,0.008910933136940002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,10240,0.08914559682210287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,10240,0.060326401392618814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,512,0.011549866199493409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,8192,0.05131733417510986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,128,0.007929599781831106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,8192,0.05212693214416504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,7168,0.045909333229064944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,8192,768,0.014416000247001648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,7168,0.046293334166208906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,6144,0.04474133253097534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,6144,0.04226133426030477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,65536,0.5911637624104817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,12288,0.12015039920806884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,5120,0.03856319983800252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,5120,0.0476149320602417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,4096,0.031052800019582112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,4096,0.034881067276000974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,16384,0.15848533312479657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,3584,0.027988266944885255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,3584,0.031067732969919843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,51200,0.4634154637654622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,3072,0.026409600178400678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,3072,0.028358399868011475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,6144,0.07062506675720215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,2560,0.025550933678944905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,2560,0.026920533180236815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,10240,0.10729066530863444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,2048,0.01629866659641266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,2048,0.024541866779327393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,7168,0.08198933601379395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,1536,0.012863999605178833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,1536,0.023051732778549196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,8192,0.09334186712900797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,1024,0.013019733627637229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,1024,0.020875734090805054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,4096,0.04617706537246704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,768,0.01016213297843933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,768,0.018616533279418944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,3072,0.037123199303944907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,5120,0.05913386742273966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,512,0.008370133241017659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,512,0.01766506632169088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,3584,0.04269440174102783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,256,0.00508480022350947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,256,0.01639359990755717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,2560,0.031224532922108965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,128,0.0044266665975252785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,128,0.016029866536458333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,2048,0.02603413263956706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,64,0.004088533421357473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,64,0.0164192001024882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,7168,32,0.004397866626580557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,7168,32,0.016639999548594155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,1536,0.020399999618530274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,1024,0.014978133638699851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,65536,0.34733012517293294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,65536,0.27145814895629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,51200,0.27909440994262696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,51200,0.21349546114603676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,16384,0.10049173037211101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,16384,0.07780373096466064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,12288,0.0757578690846761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,12288,0.06123520135879516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,256,0.008202666540940602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,128,0.007657599945863088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,10240,0.06433279911677042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,10240,0.053642666339874266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,8192,0.050963199138641356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,512,0.010718933741251628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,7168,768,0.013116799791653953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,8192,0.04608960151672363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,7168,0.046777598063151044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,7168,0.04186026652654012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,6144,0.03926933209101359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,6144,0.038652801513671876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,12288,0.11694293022155762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,5120,0.033513601620992026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,5120,0.035121067365010576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,65536,0.5773205439249675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,4096,0.028922667105992634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,16384,0.15139733950297038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,4096,0.030564266443252563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,3584,0.024488532543182374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,3584,0.02823466658592224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,51200,0.4419722557067871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,3072,0.02332906723022461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,3072,0.026662399371465046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,6144,0.06713706652323405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,2560,0.019074134031931558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,2560,0.025016532341639204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,10240,0.10325547059377034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,2048,0.015931733449300132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,2048,0.023204267024993896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,7168,0.07613653341929118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,1536,0.012963199615478515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,1536,0.02145599921544393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,8192,0.08926719824473063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,1024,0.009988266229629516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,1024,0.019562667608261107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,3072,0.035792001088460285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,768,0.008322133123874665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,768,0.017403733730316163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,4096,0.04606186548868815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,512,0.006989866495132446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,512,0.01732906699180603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,5120,0.0579584002494812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,256,0.004932266473770141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,256,0.01690453290939331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,3584,0.04123520056406657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,128,0.004726399978001913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,128,0.01629866659641266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,1536,0.018981333573659262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,64,0.00413226659099261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,64,0.016100266575813295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,6144,32,0.004285866518815359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,6144,32,0.016263467073440552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,65536,0.2806730588277181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,2048,0.025602134068806966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,65536,0.21895786921183266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,1024,0.014775466918945313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,65536,0.38185920715332033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,51200,0.22400107383728027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,2560,0.030024532477060956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,16384,0.07781546910603841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,51200,0.17171840667724608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,12288,0.05962453285853068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,16384,0.06545279820760092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,51200,0.3006239891052246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,12288,0.05333760182062784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,12288,0.07494186560312907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,10240,0.05043733517328898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,10240,0.04685333172480265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,8192,0.03850560188293457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,10240,0.06612053314844767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,8192,0.04028799931208293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,8192,0.05603946844736735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,7168,0.03661973476409912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,6144,0.030585600932439165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,7168,0.037324798107147214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,7168,0.04894826809565227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,6144,0.034209068616231283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,6144,0.04224960009256999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,5120,0.027800534168879194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,5120,0.030937600135803222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,4096,0.022989867130915324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,4096,0.027983999252319335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,4096,0.029630933205286664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,3584,0.019744000832239785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,3584,0.025809067487716674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,3584,0.02607253392537435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,3072,0.01839253306388855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,3072,0.024233599503835045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,3072,0.024277333418528238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,2560,0.014640000462532044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,2560,0.022834134101867676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,256,0.007964799801508587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,128,0.007338666419188182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,2048,0.013230933745702108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,2048,0.0216213325659434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,1536,0.0100874662399292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,512,0.010098133484522502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,1536,0.01977919936180115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,1024,0.008565333485603333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,1024,0.01759679913520813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,1024,0.010796800255775452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,768,0.007277866701285045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,6144,768,0.01260693371295929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,768,0.016551466782887776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,768,0.009727999567985535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,512,0.0057087997595469155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,512,0.008450133601824443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,256,0.0043935999274253845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,256,0.016422399878501893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,256,0.0070826664566993715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,128,0.004137599964936575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,128,0.01573013365268707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,128,0.006266666452089946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,64,0.003702399879693985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,64,0.01569919983545939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,5120,32,0.0038272000849246977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,32,0.01588053305943807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,65536,0.2243135929107666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,65536,0.18561174074808756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,5120,512,0.016659200191497803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,51200,0.17969600359598797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,51200,0.14842665990193685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,16384,0.0774943987528483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,16384,0.0640234669049581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,65536,0.3386869430541992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,12288,0.05784213145573934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,12288,0.05194880167643229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,51200,0.2639669259389242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,16384,0.09163946310679118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,10240,0.049114668369293214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,10240,0.04626986583073934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,8192,0.04006186723709106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,8192,0.040880000591278075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,16384,0.10216106573740642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,7168,0.03757013479868571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,7168,0.036500267187754315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,1536,0.014147200187047324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,6144,0.031188267469406127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,6144,0.03253973325093587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,12288,0.06903359889984131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,5120,0.027745066086451213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,5120,0.029600000381469725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,10240,0.05799359877904257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,4096,0.0219925324122111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,4096,0.026430932680765788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,8192,0.04854933420817058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,3584,0.019621332486470543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,3584,0.025094399849573772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,5120,0.03533226648966471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,3072,0.017627733945846557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,3072,0.023884799083073935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,2560,0.020823466777801513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,2560,0.015145599842071533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,2560,0.022454400857289634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,5120,2048,0.0171615997950236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,2048,0.012862933675448099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,7168,0.0448085347811381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,2048,0.020761599143346153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,2048,0.016748799880345663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,1536,0.010391466816266378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,1536,0.012771200140317282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,1536,0.019378133614857993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,1024,0.007843199868996937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,1024,0.01761066714922587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,768,0.006701866785685222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,768,0.017079466581344606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,6144,0.03727680047353109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,512,0.005497600138187409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,512,0.016696532567342125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,5120,0.03300159970919291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,256,0.00432640016078949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,256,0.015918933351834617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,4096,0.027848533789316815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,128,0.003804799914360046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,128,0.015625600020090738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,3584,0.02458560069402059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,3072,0.021704532702763877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,64,0.0034527999659379324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,64,0.015482667088508605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,4096,32,0.003651199986537298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,4096,32,0.015758933623631795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,2560,0.019362133741378785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,65536,0.20830186208089194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,65536,0.18254507382710775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,51200,0.1668992042541504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,1024,0.010154666503270467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,51200,0.14512213071187335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,768,0.00893440047899882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,16384,0.06311039924621582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,16384,0.05566720167795817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,12288,0.043195732434590656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,12288,0.048492801189422605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,512,0.008109866579373678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,10240,0.037001601854960126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,10240,0.04214613437652588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,256,0.006717866659164429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,8192,0.029700267314910888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,8192,0.03605973323186239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,4096,128,0.00602346658706665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,7168,0.02718293269475301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,7168,0.03275199929873149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,6144,0.024231467644373575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,6144,0.02937493324279785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,65536,0.3178485234578451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,5120,0.020517333348592123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,5120,0.02704319953918457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,51200,0.2580362637837728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,4096,0.016453333695729575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,16384,0.08844799995422363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,4096,0.024699733654658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,3584,0.016492799917856852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,3584,0.023965867360432942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,3584,0.023843199014663696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,3072,0.014548266927401224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,12288,0.06802666982014974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,3072,0.022781866788864135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,2560,0.012688000003496805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,2560,0.021687465906143188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,10240,0.056252801418304445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,2048,0.010851200421651204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,2048,0.020610133806864418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,8192,0.049907199541727704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,1536,0.009180800120035807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,1536,0.01875413258870443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,1536,0.012331733107566833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,1024,0.007238399982452392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,7168,0.044344532489776614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,1024,0.01760960022608439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,768,0.006086400151252747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,768,0.01733013391494751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,6144,0.038165334860483804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,512,0.005260799825191498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,512,0.01627946694691976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,5120,0.032671999931335446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,256,0.004337066908677419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,256,0.01616853376229604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,4096,0.026447999477386474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,128,0.0038176000118255613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,128,0.015294933319091797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,3072,0.020917334159215293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,64,0.0034442665676275887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,2560,0.018937599658966065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3584,32,0.003585066646337509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,64,0.015179733435312906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3584,32,0.015619200468063355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,65536,0.17798933982849122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,2048,0.01597546637058258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,65536,0.16921493212382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,51200,0.14402346611022948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,51200,0.1401365280151367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,16384,0.07914666334788004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,16384,0.05373119910558065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,1024,0.009992532928784688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,16384,0.08732159932454428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,12288,0.05929706494013468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,12288,0.043859199682871504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,768,0.00864746669928233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,10240,0.05145493348439535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,10240,0.03844266732533773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,8192,0.04052693446477254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,8192,0.03245760003725688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,512,0.007705600063006084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,7168,0.03569600184758504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,7168,0.029534933964411418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,256,0.0064746667941411335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,6144,0.03168853322664897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,6144,0.027556266387303668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3584,128,0.005941333373387655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,5120,0.02573653260866801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,5120,0.02568853298823039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,5120,0.030909866094589233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,4096,0.02111466725667318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,4096,0.023481599489847817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,65536,0.30749759674072263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,3584,0.019296000401178993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,8192,0.04642453193664551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,3584,0.022796799739201866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,7168,0.040776534875233965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,3072,0.016748799880345663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,3072,0.02159573237101237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,12288,0.06237226724624634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,2560,0.014867200454076131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,6144,0.03562133312225342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,2560,0.02064746618270874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,10240,0.05417706569035848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,2048,0.012495999534924824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,2048,0.01918826699256897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,1536,0.010416000088055929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,1536,0.018119466304779053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,4096,0.02548266649246216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,1024,0.007979733248551685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,1024,0.017723733186721803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,51200,0.2397162596384684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,768,0.006417066852251689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,768,0.017145599921544394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,512,0.005251200000445048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,3584,0.023061333100001018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,512,0.01651413341363271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,256,0.0042250668009122215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,256,0.015742933750152587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,3072,0.02018453280131022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,128,0.003667200108369192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,128,0.015402666727701821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,2560,0.01737173398335775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,64,0.0033919999996821085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,64,0.015252266327540079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,2048,0.014987732966740927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,3072,32,0.003458133339881897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,1536,0.011815466483434041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,65536,0.1453813393910726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,65536,0.15004053115844726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,1024,0.009657599528630574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,51200,0.1166805346806844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,51200,0.12078719933827717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,16384,0.04620373249053955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,16384,0.04849813381830852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,768,0.008346666892369587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,12288,0.035336534182230636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,12288,0.04002453486124675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,3072,32,0.015435733397801719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,10240,0.031083732843399048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,10240,0.03625493446985881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,512,0.007649066547552745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,8192,0.02471253275871277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,8192,0.031088000535964964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,256,0.006277333199977875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,7168,0.02211306691169739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,7168,0.028113067150115967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,3072,128,0.0056320001681645715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,6144,0.019525333245595296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,6144,0.025987199942270917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,65536,0.2756394704182943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,5120,0.017262933651606242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,5120,0.024081067244211832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,51200,0.22334613800048828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,4096,0.012299733360608418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,4096,0.0227018674214681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,16384,0.07444480260213217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,3584,0.012963199615478515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,3584,0.021542400121688843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,12288,0.05685333410898844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,3072,0.01011199951171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,3072,0.021036799748738608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,10240,0.046980265776316324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,2560,0.009155199925104777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,2560,0.019916800657908122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,8192,0.03982293208440145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,2048,0.007826133569081625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,2048,0.01835626761118571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,7168,0.035651199022928876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,1536,0.0066453332702318835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,1536,0.01752426624298096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,6144,0.031169066826502483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,1024,0.005814399818579356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,1024,0.016796799500783284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,5120,0.027313067515691118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,768,0.0047647997736930845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,768,0.016064000129699708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,4096,0.021845332781473794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,512,0.00420053352912267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,512,0.01585813363393148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,3072,0.01812160015106201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,256,0.0036949334045251214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,256,0.015659733613332113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,2048,0.013610666990280152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,128,0.0034933333595593774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,128,0.015357866883277893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,3584,0.020706133047739664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,1024,0.008996267120043437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,64,0.0031839999059836066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2560,32,0.0033290666838486993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,64,0.015188266833623251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2560,32,0.0151637335618337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,65536,0.11752106348673504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,65536,0.13414613405863446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,2560,0.016520532965660095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,51200,0.09503359794616699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,51200,0.10681706269582111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,1536,0.011161599556605022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,16384,0.04370453357696533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,16384,0.04488426844278971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,12288,0.03264426589012146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,12288,0.03654079834620158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,512,0.007522133489449819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,768,0.008390399813652038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,10240,0.02964479923248291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,128,0.005924266576766968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,10240,0.031975466012954715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,8192,0.020878932873407998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2560,256,0.006454400221506755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,8192,0.027876265843709308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,7168,0.01853760083516439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,7168,0.026285866896311443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,6144,0.01621119976043701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,6144,0.02484053373336792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,65536,0.26502720514933265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,5120,0.014097066720326743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,5120,0.024086399873097738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,51200,0.21199253400166831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,4096,0.012004266182581585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,4096,0.022219733397165934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,16384,0.0788330634435018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,3584,0.010807466506958009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,3584,0.020588799317677816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,12288,0.05957653522491455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,3072,0.009841066598892213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,3072,0.019719467560450236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,6144,0.033580799897511796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,2560,0.008920533458391826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,2560,0.01930026610692342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,10240,0.049219199021657306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,2048,0.007910400132338206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,2048,0.01821119983990987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,7168,0.036858665943145755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,1536,0.006675200164318084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,8192,0.04248426755269368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,1536,0.01716266671816508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,1024,0.005195733408133189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,1024,0.016335999965667723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,5120,0.028165332476298016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,768,0.004649599889914194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,768,0.01580586632092794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,4096,0.022706133127212525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,512,0.004092800120512644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,512,0.015777066349983215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,3072,0.019364267587661743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,256,0.004041599979003271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,256,0.015364266435305276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,3584,0.020778665939966835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,128,0.003479466587305069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,128,0.015083733201026916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,1536,0.010712533195813497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,64,0.003294933338960012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,2560,0.016099199652671814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,64,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,2048,32,0.0032960000137488045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,2048,32,0.015202132860819497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,65536,0.09486827055613199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,65536,0.12226026852925617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,1024,0.008750933408737182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,51200,0.07695146401723227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,2048,0.0132341335217158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,51200,0.09821759859720866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,16384,0.03260053396224975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,16384,0.04360853433609009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,12288,0.026678399244944258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,12288,0.034835199515024826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,512,0.007166933516661327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,10240,0.02386346658070882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,10240,0.030938667058944703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,768,0.007768533130486806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,256,0.0059125334024429325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,8192,0.019768534104029338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,8192,0.02778880000114441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,7168,0.017595734198888144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,2048,128,0.005195733408133189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,7168,0.02581226627031962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,6144,0.015460266669591268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,6144,0.02408000032107035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,65536,0.25038186709086097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,5120,0.013762133320172629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,12288,0.05399680137634277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,5120,0.02256853381792704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,4096,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,16384,0.07254506746927897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,4096,0.021154133478800456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,3584,0.009872000416119893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,3584,0.02065599958101908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,51200,0.19992213249206542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,3072,0.009038933118184407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,3072,0.01930346687634786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,6144,0.029312000672022505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,2560,0.008659199873606364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,2560,0.018782933553059898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,10240,0.04517120122909546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,7168,0.035000534852345784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,2048,0.00739519993464152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,8192,0.039086933930714926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,2048,0.01785279909769694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,1536,0.00603413333495458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,1536,0.017350399494171144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,3072,0.01646080017089844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,1024,0.005194666484991709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,1024,0.016454399625460307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,768,0.004490666588147481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,4096,0.020938666661580403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,768,0.015970133741696677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,512,0.003966933240493139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,512,0.015749333302179973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,3584,0.01911999980608622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,5120,0.026678399244944258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,256,0.0035264000296592714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,256,0.015348266561826071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,128,0.00323840007185936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,128,0.016318933169047038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,2560,0.013771733641624451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,1536,0.009756799538930256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,64,0.00308693324526151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1536,32,0.0031413334111372627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,64,0.014965333541234336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1536,32,0.01480959951877594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,65536,0.06900800069173177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,65536,0.10771733125050861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,51200,0.056903465588887536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,51200,0.08629226684570312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,2048,0.01204800009727478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,16384,0.023603200912475586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,16384,0.03631680011749268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,256,0.005751466751098633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,12288,0.018736000855763754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,512,0.0067104001839955645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,128,0.005212800204753875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,12288,0.03335466782251994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,10240,0.017151999473571777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,10240,0.03011946678161621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,768,0.0072629332542419435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,8192,0.014273066322008768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,8192,0.026824533939361572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,7168,0.017720532417297364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,7168,0.025526400407155352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1536,1024,0.008131200075149536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,6144,0.015618133544921874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,6144,0.0240938663482666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,65536,0.24778240521748862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,5120,0.014850133657455444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,5120,0.02249706586201986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,51200,0.19584213892618815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,4096,0.010971732934316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,4096,0.02205866575241089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,16384,0.07077439626057944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,3584,0.010912000139554342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,3584,0.019885865847269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,12288,0.05080639918645223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,3072,0.009617066383361817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,3072,0.019267199436823527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,10240,0.043852798144022626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,2560,0.007754666606585185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,2560,0.018605866034825645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,6144,0.03025066653887431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,8192,0.038192001978556316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,2048,0.0072970668474833175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,7168,0.03561493158340454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,2048,0.018092799186706542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,1536,0.005896533528963724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,1536,0.017046399911244712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,1024,0.0048426667849222815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,1024,0.016241066654523215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,4096,0.022165334224700926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,5120,0.026605866352717084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,768,0.004421333471934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,768,0.015681067109107973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,512,0.00395413339138031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,512,0.015565866231918335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,3584,0.019027199347813925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,256,0.0034783999125162757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,256,0.015019733707110086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,3072,0.016915200153986613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,128,0.003142400085926056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,128,0.015464533368746439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,1536,0.010082133611043294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,64,0.0030623999734719592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,64,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,1024,32,0.0030986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,2560,0.014294399817784628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,1024,32,0.01478506624698639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,65536,0.05612266858418783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,1024,0.008218666911125183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,65536,0.10098559856414795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,2048,0.01242453356583913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,51200,0.04630399942398071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,16384,0.01862186590830485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,51200,0.07990293502807617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,16384,0.034082134564717606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,12288,0.014344533284505209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,12288,0.029471999406814574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,65536,0.20047465960184732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,512,0.006570666531721751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,51200,0.16496639251708983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,768,0.007446399827798207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,256,0.0056426664193471275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,16384,0.06025813420613607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,1024,128,0.005077333251635233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,10240,0.01313706636428833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,10240,0.026589866479237872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,7168,0.01244586706161499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,8192,0.012650666634241739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,8192,0.02789120078086853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,6144,0.010857599973678588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,7168,0.025733333826065064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,6144,0.02385279933611552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,5120,0.014510933558146158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,5120,0.022587732474009196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,4096,0.011956266562143962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,4096,0.019901865720748903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,3584,0.01067733367284139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,3584,0.0195413331190745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,12288,0.04489173491795857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,3072,0.009718400239944459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,3072,0.019346133867899577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,10240,0.038423466682434085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,2560,0.007482666770617168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,8192,0.03304106593132019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,2560,0.0183786670366923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,2048,0.0065536002318064375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,6144,0.02577280004819234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,2048,0.017244799931844076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,1536,0.005618133147557576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,4096,0.017812265952428182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,1536,0.016732800006866454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,1024,0.004691199958324432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,1024,0.016426666577657064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,3584,0.015826132893562318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,768,0.004262400170167288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,768,0.01597119967142741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,7168,0.02952959934870402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,512,0.0038111999630928038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,5120,0.021717333793640138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,512,0.015520000457763672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,256,0.0033589333295822145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,256,0.014872533082962037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,3072,0.014588800072669984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,128,0.003101866692304611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,128,0.014889599879582724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,2560,0.012318933010101318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,64,0.0030016000072161358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,2048,0.010724266370137531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,768,32,0.002985599885384242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,64,0.014830933014551798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,1536,0.008910933136940002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,768,32,0.015029333035151162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,65536,0.04377066691716512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,51200,0.03618239959081014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,1024,0.007331199944019318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,65536,0.09342933495839437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,16384,0.014201600352923074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,768,0.006550399959087372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,16384,0.030763733386993408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,12288,0.011057066917419433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,12288,0.03130879998207092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,512,0.006200533111890157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,10240,0.012220799922943115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,256,0.005329066514968872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,10240,0.028424533208211263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,8192,0.010759466886520385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,8192,0.024913066625595094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,768,128,0.004993066688378652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,7168,0.01368106702963511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,51200,0.07451626459757486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,7168,0.024573866526285806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,6144,0.01223360002040863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,6144,0.022796799739201866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,51200,0.15111786524454754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,65536,0.18011946678161622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,5120,0.01267413298288981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,5120,0.020832000176111858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,4096,0.008680533369382222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,4096,0.020362667242685952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,12288,0.039578668276468915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,3584,0.009987200299898785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,3584,0.018847999970118205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,10240,0.03379093408584595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,3072,0.009058133761088053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,3072,0.018380800882975258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,8192,0.03025599916776021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,16384,0.05393813451131185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,2560,0.007143466671307881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,2560,0.01788053313891093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,2048,0.006123733520507812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,2048,0.017153066396713258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,7168,0.026476800441741943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,1536,0.005241600175698599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,1536,0.01672533353169759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,6144,0.023720532655715942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,5120,0.02021226684252421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,1024,0.00447573314110438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,1024,0.015961600343386333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,1024,0.00703359991312027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,768,0.0040853333969910945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,768,0.015570132931073507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,4096,0.016850133736928306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,512,0.0036320000886917113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,512,0.015372799833615622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,3584,0.015130666891733804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,256,0.0032757334411144257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,256,0.014878933628400167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,3072,0.013458133737246195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,128,0.0030346666773160298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,2560,0.011627733707427979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,2048,0.010046933094660442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,128,0.014476799964904785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,64,0.002828799933195114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,512,32,0.0029279999434947968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,64,0.014542933305104574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,512,32,0.014551466703414917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,65536,0.03060800035794576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,65536,0.0860970656077067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,65536,0.12782399654388427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,51200,0.02555626630783081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,51200,0.06790293057759603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,16384,0.010021332899729412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,16384,0.02809600035349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,1536,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,12288,0.011117866635322571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,12288,0.026811732848485308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,768,0.006477866570154827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,10240,0.00981760025024414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,10240,0.023944532871246337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,8192,0.008436266581217449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,512,0.006107733150323232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,8192,0.022181334098180135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,7168,0.010069333513577779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,8192,0.01913599967956543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,7168,0.021687465906143188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,256,0.005345066885153452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,6144,0.008507733543713886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,6144,0.020451200008392335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,5120,0.00791786660750707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,5120,0.02101866602897644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,12288,0.025501867135365803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,512,128,0.004948266843954722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,4096,0.00710399995247523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,4096,0.011221333344777425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,4096,0.01958079934120178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,3584,0.006853333115577698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,3584,0.018722132841746012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,3072,0.006494933366775512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,3072,0.017892267306645712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,51200,0.104912002881368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,2560,0.006248533229033152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,2560,0.017589332660039265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,16384,0.03477866649627685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,2048,0.00557226687669754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,2048,0.016845866044362386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,2048,0.007779199878374736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,1536,0.004811733464399974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,1536,0.016506666938463845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,10240,0.022603732347488404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,1024,0.0042015999555587765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,1024,0.015846400459607442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,7168,0.017706666390101114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,768,0.0038015998899936674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,768,0.01580693324406942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,768,0.005842133363087972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,6144,0.015548800428708395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,512,0.003509333233038584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,512,0.015105066696802774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,256,0.0031360000371932983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,256,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,3584,0.010539733370145162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,128,0.002842666705449422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,128,0.014843733112017313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,3072,0.010039466619491576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,64,0.0027317332724730173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,64,0.014459733168284097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,256,32,0.0027562665442625684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,2560,0.008939733107884724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,256,32,0.014594133694966635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,65536,0.023794132471084594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,65536,0.0813162644704183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,51200,0.018259199460347493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,51200,0.06334400177001953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,51200,0.08732266426086426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,16384,0.009153067072232565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,1536,0.007216000060240428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,16384,0.026589866479237872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,12288,0.00864533285299937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,12288,0.024204800526301064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,12288,0.022846933205922446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,10240,0.007666133344173431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,10240,0.022705066204071044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,1024,0.006287999947865804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,5120,0.013940266768137612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,8192,0.0070271998643875126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,8192,0.021410133441289267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,7168,0.006702933212121327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,512,0.005463466544946035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,7168,0.020946133136749267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,256,0.0049674664934476215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,6144,0.006369066735108693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,6144,0.020039467016855876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,256,128,0.004782933493455251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,5120,0.006691200037797292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,5120,0.019975467522939046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,65536,0.10925119717915852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,4096,0.00622080018122991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,4096,0.018942934274673463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,16384,0.029173332452774047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,3584,0.00654720018307368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,3584,0.018514132499694823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,3584,0.010019200046857198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,3072,0.00622506688038508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,3072,0.018500266472498576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,3072,0.00918826659520467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,10240,0.019709867238998414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,2560,0.006121600170930227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,2560,0.008654933174451191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,2048,0.005350400010744731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,2560,0.017195733388264973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,2048,0.007482666770617168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,2048,0.016770132382710776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,1536,0.004755199948946635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,8192,0.016659200191497803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,1024,0.004065066576004028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,1024,0.016355199615160625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,1024,0.006202666461467743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,768,0.0037205333511034647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,7168,0.015339733163515726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,768,0.01566506624221802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,512,0.003437866767247518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,512,0.015170133113861084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,256,0.003052799900372823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,512,0.00537066658337911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,256,0.01490133305390676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,128,0.0028543998797734577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,128,0.01463573376337687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,6144,0.013416533668835958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,128,0.004759466648101807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,64,0.0027061333258946735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,64,0.014665599664052328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,128,32,0.0027903998891512555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,32,0.014593066771825156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,65536,0.07934719721476237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,51200,0.012338133653004964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,51200,0.05968426863352457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,16384,0.007784533500671387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,128,1536,0.016150400042533875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,12288,0.0069482664267222095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,16384,0.02659733295440674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,10240,0.006567466755708058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,12288,0.023640533288319908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,8192,0.006723199784755707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,10240,0.022740266720453896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,5120,0.011974400281906128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,8192,0.020906666914621987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,7168,0.006461866696675618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,6144,0.006075733403364817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,5120,0.0064629331231117245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,7168,0.020999467372894286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,6144,0.020147200425465903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,4096,0.006122666597366333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,5120,0.019796266158421835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,4096,0.019100799163182577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,3584,0.0064085334539413456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,3584,0.018133334318796792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,3072,0.006141866743564606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,3072,0.017825067043304443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,2560,0.01779306729634603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,2048,0.005323733389377594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,4096,0.010241066416104633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,2048,0.017063466707865398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,1024,0.004084266722202301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,1536,0.016313599546750386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,768,0.003638399889071783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,1024,0.016123732924461363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,512,0.0034133332471052804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,768,0.015589333573977151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,256,0.002976000060637792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,512,0.015710933009783427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,128,0.002902399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,256,0.015144532918930054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,64,0.002678400029738744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,128,0.014633599917093912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,32,0.0027722666660944624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,64,0.01453439990679423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,65536,0.013884799679120383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,64,32,0.014549332857131957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,51200,0.0116266667842865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,65536,0.07776640256245931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,16384,0.008022400240103405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,51200,0.059324800968170166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,12288,0.007228800157705943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,16384,0.026044799884160356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,12288,0.023483733336130776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,1536,0.007046400010585785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,10240,0.02257919907569885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,8192,0.006757333377997081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,8192,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,7168,0.006450133522351582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,7168,0.021617066860198975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,6144,0.00613973339398702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,2560,0.006052266558011373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,768,0.005750399827957153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,5120,0.006405333181222279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,6144,0.020001065731048585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,1536,0.004678399860858917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,4096,0.005970133344332377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,5120,0.020117332537968956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,3584,0.006494933366775512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,4096,0.019820799430211387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,3584,0.018795732657114664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,3072,0.0061471998691558834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,257,128,256,0.0050357331832249965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,3072,0.018407466014226277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,2048,0.005343999962011973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,1536,0.00472320020198822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,2560,0.01744426687558492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,64,65536,0.017884800831476845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,2048,0.016992000738779704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,1024,0.003990400085846583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,768,0.0036938667297363283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,1536,0.016334933042526246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,512,0.0032960000137488045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,1024,0.01569386621316274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,256,0.0030016000072161358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,768,0.015361066659291586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,128,0.0028234665592511495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,512,0.01532373329003652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,64,0.0026581334571043652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,256,0.01514346698919932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,32,0.002696533252795537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,128,0.014616533120473226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,64,0.01437226633230845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,257,32,32,0.014668800433476768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,10240,0.006697600086530049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,16384,0.4412490526835124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,257,32,2560,0.006050133208433787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,16384,0.799343999226888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,12288,0.34270188013712566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,12288,0.6198794682820638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,51200,1.3236896514892578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,10240,0.29342079162597656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,10240,0.5146602630615235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,51200,2.089379247029622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,8192,0.4133301417032878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,8192,0.2399189313252767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,10240,0.36289599736531575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,51200,2.5680501302083334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,7168,0.23799039522806803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,7168,0.21565866470336914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,6144,0.3182293256123861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,7168,0.38122987747192383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,6144,0.21171199480692543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,5120,0.2685951868693034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,5120,0.1636544068654378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,5120,0.19143253962198895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,4096,0.14062506357828777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,3584,0.1969194730122884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,16384,0.5197056134541829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,3584,0.12560106913248698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,3584,0.12977279822031657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,3072,0.11154773235321044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,3072,0.17385066350301107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,2560,0.14677546819051107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,2560,0.09696319897969564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,12288,0.4070677439371745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,2048,0.12068373362223309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,2048,0.08366826375325521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,2048,0.08820266723632812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,1536,0.09507733186086019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,1536,0.06824959913889567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,1024,0.06631040175755819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,1024,0.05537600119908651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,4096,0.24863893191019693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,1024,0.05070826609929403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,768,0.06012800137201944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,768,0.048325332005818684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,512,0.04118613402048747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,768,0.04873600006103516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,512,0.04225920041402181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,256,0.029586132367451983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,256,0.03476800123850505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,8192,0.2640373388926188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,128,0.02453226645787557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,128,0.03064746658007304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,6144,0.2064906597137451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,64,0.01827413241068522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,64,0.029226666688919066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,32,0.016682666540145875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,32,0.031370667616526286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,4096,0.14810986518859864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,3072,0.11189866860707601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,2560,0.09440639813741049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,1536,0.06680959860483805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,512,0.0364682674407959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,256,0.031623466809590654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,128,0.028284800052642823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,65536,1.399619166056315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,51200,1.0784437815348307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,16384,0.4027317365010579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,65536,2.7290239969889325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,51200,2.066706085205078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,12288,0.4957589467366536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,12288,0.31402772267659507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,10240,0.4124138514200847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,16384,0.648695437113444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,10240,0.26329280535380045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,8192,0.3330538749694824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,8192,0.19474879900614422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,7168,0.29503892262776693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,7168,0.1747242609659831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,6144,0.2560010592142741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,6144,0.15467519760131837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,12288,0.34203945795694984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,5120,0.24186879793802896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,65536,1.912453333536784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,5120,0.13285120328267414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,4096,0.17563625971476238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,4096,0.11191999912261963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,16384,0.43506879806518556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,3584,0.17612266540527344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,3584,0.10265493392944336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,10240,0.2678880055745443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,51200,1.3457941691080728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,3072,0.1480639934539795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,6144,0.16374400456746419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,2560,0.1157258669535319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,3072,0.1008010705312093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,2560,0.0794975996017456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,2048,0.09512426853179931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,2048,0.06841599941253662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,7168,0.19447679519653321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,1536,0.07471040089925131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,1536,0.06406399806340536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,8192,0.21983572642008462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,1024,0.06047360102335612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,1024,0.04853013356526693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,3072,0.0852661371231079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,768,0.0447925329208374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,768,0.043348264694213864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,4096,0.11565226713816326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,512,0.038916265964508055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,512,0.0382261315981547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,1536,0.049703466892242434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,256,0.02496853272120158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,256,0.032219733794530234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,5120,0.15174080530802408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,128,0.020850133895874024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,2560,0.07790186405181884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,128,0.030418133735656737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,64,0.017462400595347087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,64,0.030230400959650676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,3584,0.10104107062021891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,51200,32,0.017819732427597046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,51200,32,0.03293333252271016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,65536,0.4707626660664876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,65536,0.7934762954711914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,1024,0.04298666715621948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,2048,0.06300479968388875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,51200,0.37202987670898435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,51200,0.6348629633585612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,16384,0.20572586059570314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,16384,0.15291627248128253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,12288,0.1597962697347005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,12288,0.10761600335439045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,256,0.022177066405614218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,128,0.019501866896947224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,10240,0.1314794699350993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,10240,0.09122666517893473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,65536,0.554028828938802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,51200,0.4483327865600586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,8192,0.10787093639373779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,8192,0.08069012959798177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,7168,0.09447786808013917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,8192,0.08032746315002441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,512,0.027677865823109944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,7168,0.07116693655649821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,7168,0.06382826566696168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,6144,0.08542293707529704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,6144,0.0634389321009318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,5120,0.07004160086313883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,5120,0.05750720103581747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,4096,0.05769280195236206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,16384,0.13548800150553386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,4096,0.04955413341522217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,3584,0.054915201663970944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,3584,0.045390931765238444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,3072,0.04610133171081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,3072,0.041442131996154784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,12288,0.10475413004557292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,2560,0.039426132043202715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,51200,768,0.033810134728749594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,2560,0.03668160041173299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,2048,0.033019733428955075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,2048,0.032382933298746745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,2048,0.025624533494313557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,10240,0.09232319990793864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,1536,0.026589866479237872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,1536,0.027553067604700728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,6144,0.055213868618011475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,1024,0.018603734175364175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,5120,0.04780160188674927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,1024,0.023922133445739745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,768,0.014849066734313965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,4096,0.03970666726430257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,768,0.02262079914410909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,512,0.011196800072987874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,3584,0.036405332883199054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,512,0.021061333020528157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,256,0.00864746669928233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,512,0.013069867094357809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,256,0.011180800199508668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,256,0.017933867375055947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,128,0.006156800190607706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,3072,0.03247893253962199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,128,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,128,0.0100682665904363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,64,0.005458133419354757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,32,0.005809066692988077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,64,0.017858133713404337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,32,0.017998933792114258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,2560,0.028701867659886676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,65536,0.3674432118733724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,51200,0.46015040079752606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,65536,0.5846826553344726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,16384,0.16373440424601238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,16384,0.11963520050048829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,51200,0.2917279879252116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,12288,0.11830933094024658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,12288,0.08707839647928874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,10240,0.09895359675089518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,10240,0.07455573081970215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,1536,0.021010132630666097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,8192,0.07988479932149252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,8192,0.06262933413187663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,1024,0.01649066706498464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,768,0.015336533387502035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,7168,0.07096532980600992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,7168,0.05712639888127645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,6144,0.06239680051803589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,6144,0.05139840046564738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,65536,0.4779061317443848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,5120,0.06035306851069132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,5120,0.045857067902882895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,16384,0.10947306950887044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,51200,0.3865983963012695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,4096,0.04375360012054443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,4096,0.041665065288543704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,12288,0.08656533559163412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,3584,0.039574400583903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,3584,0.03778880039850871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,3072,0.03477866649627685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,3072,0.034185600280761716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,3072,0.026179200410842894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,10240,0.07249173323313395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,2560,0.030130134026209517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,2560,0.03059200048446655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,2048,0.025068799654642742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,2048,0.026950399080912273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,8192,0.05762666861216227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,1536,0.01997119983037313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,1536,0.02448319991429647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,7168,0.052323198318481444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,1024,0.013770666718482972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,1024,0.021875200668970744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,6144,0.04525119860967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,768,0.011432533462842304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,768,0.020964266856511433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,5120,0.03901013135910034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,512,0.009115733702977498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,512,0.01930453379948934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,4096,0.032237867514292404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,1024,0.014085333546002707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,256,0.006705066561698914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,256,0.017308799425760905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,3584,0.029361067215601604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,128,0.005162666738033295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,128,0.016567466656366985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,64,0.004759466648101807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,64,0.016902399063110352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,32,0.005047466854254405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,32,0.01727573275566101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,2560,0.023915733893712363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,2048,0.020193066199620566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,1536,0.016369066635767617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,65536,0.35363734563191734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,65536,0.5384991963704426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,51200,0.25490986506144203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,768,0.01206826666990916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,51200,0.4309162775675456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,16384,0.13008000055948893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,16384,0.09590506553649902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,12288,0.10065066814422607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,12288,0.07647893428802491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,10240,0.08433813254038493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,10240,0.06589546600977579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,512,0.010630399982134501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,256,0.008572799960772197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,8192,0.06821653048197428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,128,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,8192,0.05633066495259603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,7168,0.0612607995669047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,7168,0.051514665285746254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,6144,0.05402773221333822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,6144,0.046804265181223555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,12288,0.0780074675877889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,5120,0.04565973281860351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,65536,0.4343263943990071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,5120,0.04163519938786824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,4096,0.03783786694208781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,4096,0.03678613503774007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,16384,0.10577279726664227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,3584,0.03452479839324951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,3584,0.03454720179239909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,51200,0.35663681030273436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,3072,0.03395946820576985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,3072,0.031276800235112506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,10240,0.0686463991800944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,2560,0.02635306715965271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,2560,0.02863466739654541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,7168,0.04642346700032552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,2048,0.02211839954058329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,2048,0.025722666581471758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,8192,0.05217599868774414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,1536,0.017499732971191406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,1536,0.023363200823465984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,6144,0.04033279816309611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,1024,0.012614400188128153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,1024,0.021996800104777017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,3072,0.023490132888158162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,768,0.011015466849009196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,768,0.020733867088953653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,4096,0.02916266719500224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,512,0.009034666419029235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,512,0.01845973332722982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,5120,0.03709226846694946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,256,0.006361599763234456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,256,0.0179584006468455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,3584,0.0266485333442688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,128,0.005106133222579956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,128,0.016551466782887776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,2560,0.02148800094922384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,1536,0.01451520025730133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,64,0.004712533454100291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,64,0.016722132762273155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,32,0.005018666883309682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,32,0.016777600844701132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,65536,0.2719189325968424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,65536,0.4199498812357585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,2048,0.01835093299547831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,51200,0.3306528091430664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,65536,0.28140907287597655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,51200,0.21479999224344887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,16384,0.10446186860402425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,16384,0.08169386386871338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,12288,0.08137919902801513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,16384,0.07286826769510904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,12288,0.06607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,10240,0.0680191993713379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,10240,0.057366398970286045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,10240,0.04757866859436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,8192,0.055396266778310145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,1024,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,8192,0.048444799582163495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,7168,0.04931733210881551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,7168,0.044350934028625486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,7168,0.035417600472768145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,6144,0.043494399388631186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,6144,0.0401472012201945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,512,0.009090133508046468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,128,0.007231999933719635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,256,0.00767573316891988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,51200,0.23457493782043456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,5120,0.036638931433359785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,4096,0.0346997340520223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,12288,0.05648106733957926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,4096,0.03262400031089783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,768,0.01097706655661265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,3584,0.027837866544723512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,3584,0.03060479958852132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,3072,0.024613332748413087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,3072,0.027050666014353436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,2560,0.021265067656834922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,2560,0.025312000513076784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,2560,0.017008000612258913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,2048,0.017433599630991618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,2048,0.023426133394241332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,8192,0.038789331912994385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,1536,0.013184000054995218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,1536,0.021898667017618813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,6144,0.03128746747970581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,1024,0.010133333007494609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,5120,0.037628801663716634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,1024,0.020029866695404054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,768,0.00871573289235433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,768,0.018261333306630455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,5120,0.027569067478179932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,4096,0.023337600628534953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,512,0.007354666789372762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,3584,0.02140586574872335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,3072,0.019403733809789023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,512,0.017122133572896322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,256,0.005194666484991709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,256,0.016549332936604818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,128,0.004513066510359446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,128,0.0158869336048762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,2048,0.014995200435320535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,64,0.004217599829037985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,1536,0.01304746667544047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,64,0.01625386675198873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,32,0.0044832001129786175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,32,0.016821332772572837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,65536,0.344215456644694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,1024,0.011129599809646607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,65536,0.24653120040893556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,51200,0.19528746604919434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,51200,0.27314453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,16384,0.09162666797637939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,16384,0.07975359757741293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,16384,0.0670528014500936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,12288,0.07110826969146729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,12288,0.06027413209279379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,10240,0.06003413200378418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,10240,0.05283946593602499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,768,0.010038399696350097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,512,0.00827519992987315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,8192,0.04864426851272583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,256,0.007420800129572551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,8192,0.04508053461710612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,7168,0.04350293477376302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,7168,0.04204266468683879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,6144,0.03858453432718913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,128,0.007166933516661327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,6144,0.03764479955037435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,5120,0.03339626789093018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,5120,0.03405119975407918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,65536,0.2562741279602051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,4096,0.030642133951187134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,51200,0.2116703987121582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,12288,0.05105706850687662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,3584,0.024638932943344117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,3584,0.027411200602849323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,10240,0.04332799911499023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,3072,0.022295467058817544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,3072,0.025484800338745117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,4096,0.029912533362706502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,2560,0.018866133689880372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,8192,0.03642239967981974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,2560,0.024086399873097738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,2048,0.01534293293952942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,2048,0.02252266605695089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,7168,0.03281919956207276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,1536,0.01204159955183665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,6144,0.028678399324417115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,1536,0.02061226765314738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,1024,0.00920853316783905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,1024,0.019205333789189656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,5120,0.02458666761716207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,768,0.008014933268229166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,768,0.01752106746037801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,768,0.008642133076985676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,512,0.0065536002318064375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,512,0.017148800690968833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,4096,0.020938666661580403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,256,0.00489279975493749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,256,0.016312533617019655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,3584,0.019155200322469076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,128,0.004362666606903076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,128,0.015756799777348836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,3072,0.0173962672551473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,64,0.003870933254559835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,64,0.016429866353670754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,32,0.004141866664091746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,32,0.016452266772588094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,65536,0.3148106575012207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,2560,0.015901866555213928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,2048,0.01384106675783793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,1536,0.011817600329717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,65536,0.23704106012980142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,1024,0.009956266482671101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,51200,0.24848640759785973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,51200,0.17862613995869953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,16384,0.08112640380859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,12288,0.0634335994720459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,512,0.007154133419195812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,16384,0.06010453303654989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,10240,0.05528853336970011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,12288,0.0558837334314982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,10240,0.04974933465321858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,8192,0.04275946617126465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,8192,0.041810135046641037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,256,0.0065738668044408154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,7168,0.0384607990582784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,7168,0.03852479855219523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,16384,0.06896426677703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,6144,0.03422613143920898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,128,0.006192000210285186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,6144,0.03538666566212972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,65536,0.23711892763773598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,5120,0.031082665920257567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,5120,0.031152000029881794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,51200,0.19824426968892414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,12288,0.04655040105183919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,4096,0.027315199375152588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,4096,0.02709653377532959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,3584,0.02175253431002299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,3584,0.02611306707064311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,10240,0.0396938681602478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,3072,0.019268266359965005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,3072,0.0242741326491038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,8192,0.03269546627998352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,2560,0.0164490669965744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,2560,0.023507199684778848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,7168,0.02988053361574809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,2048,0.013006933530171714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,2048,0.021448532740275063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,1536,0.010905599594116211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,6144,0.026230400800704955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,1536,0.020574933290481566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,1536,0.010934399565060933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,1024,0.018260266383488974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,5120,0.022655999660491942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,768,0.007347199817498524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,768,0.017496534188588462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,1024,0.008400000135103862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,512,0.005714133381843567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,512,0.017003732919692992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,4096,0.019401599963506065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,256,0.0046528001626332605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,3584,0.017578667402267455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,256,0.016662399967511496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,128,0.004106666644414266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,128,0.01568000018596649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,3072,0.015585066874821982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,64,0.0038176000118255613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,64,0.015946666399637856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,32,0.003949866692225138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,32,0.01609280010064443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,2560,0.014782933394114175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,65536,0.21153386433919272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,2048,0.012758400042851767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,51200,0.22775893211364745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,51200,0.16810773213704427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,1024,0.009317333499590557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,16384,0.06902080376942953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,16384,0.062352001667022705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,16384,0.05401386817296346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,12288,0.05527253150939941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,12288,0.05161813497543335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,65536,0.28461227416992185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,10240,0.052894934018452966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,10240,0.046972799301147464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,768,0.007716266810894013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,8192,0.037323733170827225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,8192,0.038711468378702804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,512,0.007006933291753133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,256,0.0063360000650088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,7168,0.03398720026016235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,7168,0.03562239805857341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,6144,0.030404265721638995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,128,0.0061589335401852924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,6144,0.033504001299540204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,5120,0.02821439901987712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,5120,0.029660799105962116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,65536,0.2243882656097412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,4096,0.02249493400255839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,4096,0.02648853262265523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,51200,0.18392106691996257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,3584,0.019413334131240845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,3584,0.024975999196370443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,12288,0.04204053481419881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,3072,0.017056000232696534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,3072,0.02358186642328898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,10240,0.03598080078760783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,2560,0.01446399986743927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,2560,0.022528000672658286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,8192,0.02963520089785258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,2048,0.01202239990234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,7168,0.026970666646957398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,2048,0.02099626660346985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,1536,0.0100874662399292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,6144,0.023358933130900063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,1536,0.019269333283106486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,1024,0.007704533139864604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,1024,0.017968000968297322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,5120,0.020075732469558717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,768,0.006951466699441274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,768,0.016646400094032288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,4096,0.017246933778127034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,512,0.005464533468087515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,512,0.016860800981521606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,3584,0.015660799543062844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,256,0.0044821331898371375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,256,0.01586666703224182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,3072,0.014407466848691305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,128,0.003975466638803482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,2560,0.013338667154312134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,128,0.015508266290028891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,2048,0.0116757333278656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,64,0.00367253323396047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,1536,0.010037333766619364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,64,0.0159061332543691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,32,0.0039327998956044516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,1024,0.0079925333460172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,32,0.0160778671503067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,768,0.007210666437943776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,65536,0.2225503921508789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,51200,0.17060906092325848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,65536,0.18402773539225262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,16384,0.06326826810836791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,51200,0.14649813969930012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,16384,0.05788373152414957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,12288,0.04880106846491496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,12288,0.048403199513753256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,512,0.006662400066852569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,10240,0.0443125327428182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,10240,0.04521919886271159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,256,0.006050133208433787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,8192,0.0322869340578715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,8192,0.03671786785125732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,7168,0.029182932774225873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,7168,0.034075733025868735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,128,0.005625600119431814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,6144,0.02804800073305766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,6144,0.030694399277369184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,65536,0.20590079625447594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,51200,0.16111680666605632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,5120,0.023654399315516154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,5120,0.028089600801467895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,4096,0.019814399878184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,4096,0.02587626576423645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,16384,0.04763413270314534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,12288,0.03911573489507039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,3584,0.01720213294029236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,3584,0.024375466505686443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,3072,0.014781866470972696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,3072,0.022600533564885457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,10240,0.035259731610616046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,2560,0.012708266576131185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,2560,0.021589332818984987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,8192,0.026291199525197345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,2048,0.010875733693440755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,2048,0.020692267020543418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,7168,0.02379946708679199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,1536,0.00895146628220876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,1536,0.019268266359965005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,6144,0.020988800128300986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,1024,0.007481599847475688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,1024,0.01781546672185262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,5120,0.01844266653060913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,768,0.006241066753864289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,768,0.016702934106191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,4096,0.015878400206565856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,512,0.004811733464399974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,512,0.016611199577649435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,3072,0.01362879971663157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,256,0.004120533416668574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,256,0.01567893326282501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,3584,0.014656000336011252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,2048,0.011187199751536052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,128,0.0036490666369597114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,128,0.015400532881418863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,64,0.003522133330504099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,1024,0.007696000238259633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,64,0.01546239952246348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,32,0.0036288000643253326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,32,0.015621333320935567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,65536,0.19371093114217122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,2560,0.012705066800117492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,51200,0.1530058701833089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,51200,0.13556373914082845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,1536,0.009381332993507385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,16384,0.05644479990005493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,65536,0.1739242712656657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,16384,0.05538986523946127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,12288,0.04650239944458008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,12288,0.04531840085983276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,256,0.006205866734186808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,10240,0.036659200986226395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,10240,0.0396234671274821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,512,0.0065290664633115125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,8192,0.029494400819142657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,768,0.007214933137098948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,8192,0.03480319976806641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,7168,0.02704746723175049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,7168,0.03205440044403076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,128,0.0058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,6144,0.024046933650970458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,6144,0.029682133595148725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,65536,0.18989334106445313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,5120,0.023081600666046143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,5120,0.026257065931955974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,12288,0.035646935304005936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,4096,0.01667733391125997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,4096,0.024387200673421226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,51200,0.1554197311401367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,16384,0.04594666560490926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,3584,0.01520639955997467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,3584,0.023246934016545616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,10240,0.030450133482615153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,3072,0.014126933614412942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,3072,0.022177066405614218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,7168,0.023041067520777385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,6144,0.019631999731063842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,2560,0.011553066968917846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,2560,0.020753065745035805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,2048,0.010010666648546855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,2048,0.019828265905380248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,8192,0.025197867552439374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,1536,0.008309333523114523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,1536,0.018626133600870766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,1024,0.0067210664351781205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,1024,0.017758933703104655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,4096,0.014879999558130899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,768,0.005221333106358846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,768,0.016454399625460307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,5120,0.01730239987373352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,512,0.0045045331120491024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,512,0.015915733575820924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,3584,0.013860266407330832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,256,0.003894400099913279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,256,0.015619200468063355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,3072,0.012820266683896384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,128,0.003576533248027166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,128,0.015016532937685647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,1536,0.00855573316415151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,64,0.0034954667091369627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,64,0.015281066298484802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,32,0.003502933432658514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,32,0.015525333086649575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,2560,0.011907200018564861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,65536,0.16783787409464518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,65536,0.15988267262776693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,51200,0.13369812965393066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,2048,0.010557867089907328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,51200,0.12614719867706298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,16384,0.04982399940490723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,16384,0.05217386484146118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,1024,0.0076330666740735366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,12288,0.04330346584320068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,12288,0.04350613355636597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,512,0.006260266900062561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,256,0.005876266459623972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,10240,0.03570346832275391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,10240,0.039078398545583086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,128,0.005576533575852713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,768,0.006780800223350525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,8192,0.026884265740712482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,7168,0.02456106742223104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,8192,0.03246826728185018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,7168,0.029693865776062013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,6144,0.022460800409317017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,6144,0.027587199211120607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,65536,0.17558719317118326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,12288,0.03411413431167602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,5120,0.02531840006510417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,16384,0.043295999368031815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,4096,0.015603199601173401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,4096,0.023797333240509033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,51200,0.14224960009256998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,3584,0.013713066776593527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,3584,0.02231360077857971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,5120,0.020413867632548013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,3072,0.012452266613642375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,3072,0.0213919997215271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,6144,0.018492799997329713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,2560,0.010987733801205952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,2560,0.0206058661142985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,10240,0.028510934114456175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,2048,0.009378133217493693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,2048,0.019156267245610557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,8192,0.02390506664911906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,1536,0.00795306662718455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,1536,0.018011732896169027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,7168,0.021501866976420085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,1024,0.006128000219662985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,1024,0.016720000902811685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,4096,0.014056533575057983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,768,0.005133866767088572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,768,0.0159850666920344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,3072,0.012007466952006022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,5120,0.016424533724784852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,512,0.004244266450405121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,3584,0.01309440036614736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,512,0.01585813363393148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,256,0.0036789332826932275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,256,0.015594666202863058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,128,0.003316266586383184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,128,0.015246933698654175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,2560,0.011192533373832702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,64,0.0032448001205921174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,1536,0.007875200112660725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,32,0.0033973333736260734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,64,0.015133866667747497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,32,0.015410133202870688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,65536,0.14393173853556315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,65536,0.1493578592936198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,2048,0.009801600376764934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,51200,0.11575466791788738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,51200,0.11851627031962078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,1024,0.007043200234572093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,16384,0.04355413516362508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,16384,0.0482752005259196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,256,0.005471999943256378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,128,0.005182933310667674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,12288,0.035916801293691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,12288,0.03887893358866374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,512,0.00588266650835673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,10240,0.03272106647491455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,10240,0.0370527982711792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,8192,0.023144533236821495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,768,0.006357333560784657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,8192,0.030746666590372722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,7168,0.02094399929046631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,7168,0.028358399868011475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,6144,0.018012799819310508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,6144,0.026632533470789595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,65536,0.15714559555053711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,5120,0.018170666694641114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,5120,0.024701867500940958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,51200,0.13356800079345704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,4096,0.014095999797185264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,4096,0.022685867547988892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,16384,0.04123733441034953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,3584,0.012866133451461792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,3584,0.021600000063578286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,12288,0.03179840048154195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,3072,0.011623467008272808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,3072,0.021167999505996703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,6144,0.01772693395614624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,10240,0.027436800797780353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,2560,0.010193066795667012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,2560,0.01958079934120178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,2048,0.008789333701133727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,2048,0.017948800325393678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,7168,0.02033066749572754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,1536,0.0075328002373377485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,1536,0.017230933904647826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,8192,0.02294293244679769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,1024,0.005351466437180838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,1024,0.016203733285268147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,5120,0.016100266575813295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,768,0.004763733347256978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,768,0.01623679995536804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,3072,0.011751466989517212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,512,0.004078933348258337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,4096,0.013558399677276612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,512,0.015763200322786965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,256,0.0035648000737031303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,256,0.015335466464360556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,3584,0.012934399644533792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,128,0.0033781332274278007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,128,0.01504533290863037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,1536,0.007749333480993907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,64,0.0031456001102924346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,2560,0.01069119970003764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,32,0.0032960000137488045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,32,0.015244799852371215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,65536,0.11695679823557537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,1024,0.0068917334079742435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,65536,0.13338346481323243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,2048,0.009360000491142273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,51200,0.09443840185801188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,51200,0.10546346505482991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,16384,0.03717546860376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,16384,0.04687360127766927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,64,0.014937600493431092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,12288,0.034533333778381345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,12288,0.037827201684316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,256,0.005490133166313171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,10240,0.030528000990549725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,10240,0.03363093137741089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,768,0.006414933502674103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,512,0.005854933460553487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,8192,0.01873813271522522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,128,0.005329066514968872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,8192,0.029237333933512372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,7168,0.019513599077860513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,7168,0.026790400346120197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,6144,0.017946666479110716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,6144,0.025060266256332397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,65536,0.15014185905456542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,5120,0.016821332772572837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,5120,0.023235199848810832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,51200,0.12194240093231201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,4096,0.010722133517265319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,4096,0.02254293362299601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,16384,0.03540053367614746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,3584,0.01092693308989207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,3584,0.021197867393493653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,12288,0.028100266059239702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,3072,0.00918506681919098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,3072,0.01981226603190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,10240,0.025153066714604693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,2560,0.008549333612124125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,2560,0.018454400698343913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,7168,0.01799573302268982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,8192,0.019346133867899577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,2048,0.007181866466999054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,6144,0.015786666671435037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,2048,0.017753599087397258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,1536,0.006239999830722809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,1536,0.01686720053354899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,5120,0.01437333325544993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,1024,0.004961066444714864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,1024,0.016427733500798545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,4096,0.01269760032494863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,768,0.004411733150482178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,768,0.016150400042533875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,512,0.004183466732501984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,3072,0.011075199643770853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,512,0.01565226713816325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,256,0.0035413332283496858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,256,0.015212800105412802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,3584,0.011955199639002483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,128,0.003272533416748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,128,0.014776532848676046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,2048,0.008374399940172831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,64,0.0031701333820819853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,64,0.014963199694951376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,32,0.0034133332471052804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,32,0.015313067038853965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,65536,0.0917194684346517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,2560,0.009829333424568177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,65536,0.1203925371170044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,1536,0.007500799993673961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,51200,0.07407039801279704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,51200,0.09490133126576741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,16384,0.030315732955932616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,16384,0.040557865301767984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,1024,0.00652159998814265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,12288,0.02643306652704875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,12288,0.032690133651097616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,768,0.00610453337430954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,10240,0.023188267151514688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,256,0.0053247998158137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,512,0.005787733197212219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,10240,0.029677865902582805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,8192,0.01718719998995463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,8192,0.026200532913208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,128,0.005155199766159057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,7168,0.01562346617380778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,7168,0.025514666239420575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,6144,0.013801599542299906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,6144,0.023923200368881226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,65536,0.14066880544026691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,5120,0.014806399742762247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,5120,0.02272640069325765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,51200,0.11323200066884358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,16384,0.037140266100565596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,12288,0.02829759915669759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,4096,0.011245866616566975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,4096,0.019886932770411172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,3584,0.009734400113423665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,3584,0.019663999478022255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,3072,0.009289600451787313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,3072,0.019471999009450278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,6144,0.01640959978103638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,2560,0.007912533481915791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,2560,0.01863893270492554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,7168,0.018905599912007652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,2048,0.006931200126806895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,2048,0.018167465925216675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,10240,0.02450666626294454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,1536,0.0060362666845321655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,8192,0.020447999238967896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,1536,0.017203199863433837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,1024,0.0049792001644770306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,1024,0.016540799538294473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,3072,0.010193066795667012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,768,0.004229333500067393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,768,0.016102400422096253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,4096,0.012625066439310708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,512,0.0038058665891488397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,5120,0.014716800053914389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,512,0.015460266669591268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,256,0.0033802665770053864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,256,0.015027200182278952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,3584,0.01138879954814911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,128,0.0030954666435718536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,1536,0.0073738664388656614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,128,0.0150709331035614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,64,0.0029440000653266907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,64,0.014833066860834757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,32,0.0030805334448814393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,2560,0.009527466694513957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,32,0.015117866794268289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,65536,0.0671498696009318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,65536,0.10685546398162842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,2048,0.008488532900810242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,1024,0.006569600105285645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,51200,0.054780801137288414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,51200,0.0843989372253418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,16384,0.03630826473236084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,16384,0.024769065777460735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,12288,0.01817493240038554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,256,0.005305600166320801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,12288,0.03243626753489177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,768,0.006121600170930227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,10240,0.016341333587964378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,128,0.005141333242257436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,10240,0.030161066850026445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,512,0.005746133128801982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,8192,0.012771200140317282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,8192,0.025448532899220784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,7168,0.011990400155385335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,7168,0.024408533175786336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,6144,0.014201600352923074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,6144,0.02250880002975464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,65536,0.11803306738535564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,5120,0.013377066453297934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,51200,0.09387946923573812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,5120,0.02139520049095154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,4096,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,4096,0.021950932343800862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,16384,0.030829866727193195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,3584,0.009917866190274556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,3584,0.020425599813461304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,12288,0.024566400051116943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,3072,0.00894719958305359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,3072,0.018935465812683107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,10240,0.02127359906832377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,2560,0.007842133442560833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,8192,0.017831466595331826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,2560,0.01842986742655436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,2048,0.006822399795055389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,2048,0.01774719953536987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,7168,0.016479999820391337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,1536,0.005891199906667074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,1536,0.01697280009587606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,6144,0.014215466380119324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,1024,0.004733866453170777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,1024,0.016300800442695617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,5120,0.012545067071914672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,768,0.00420053352912267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,768,0.015822933117548624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,4096,0.010732799768447876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,512,0.0037162666519482933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,512,0.01556373337904612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,3584,0.01033066709836324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,256,0.0032831999162832894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,256,0.014987732966740927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,3072,0.009530666470527648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,128,0.002961066613594691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,128,0.014654933412869772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,1536,0.007334400216738384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,2560,0.008963200449943542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,64,0.0029237332443396253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,32,0.002980266759792964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,64,0.014618666966756186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,1024,0.0062720000743865965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,768,0.00580266664425532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,65536,0.05494399865468343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,51200,0.0453439990679423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,65536,0.10006612936655682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,51200,0.07893973191579183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,16384,0.018222934007644652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,16384,0.032679466406504314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,512,0.005496533215045929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,12288,0.014915200074513755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,2048,0.007875200112660725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,12288,0.030290132761001586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,10240,0.014017066359519959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,10240,0.02744106650352478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,256,0.005103999873002371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,8192,0.01225920021533966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,8192,0.02490133245786031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,32,0.014765866597493491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,7168,0.011752532919247945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,7168,0.023820799589157105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,128,0.00484799991051356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,6144,0.012162133057912191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,6144,0.022022400299708048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,65536,0.11632106304168702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,5120,0.011005866527557372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,5120,0.02105600039164225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,51200,0.09168533484141031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,4096,0.00942080020904541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,16384,0.030732800563176472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,4096,0.019610667228698732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,3584,0.008365866541862488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,12288,0.023884799083073935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,10240,0.020641066630681357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,3072,0.0076330666740735366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,3072,0.018523732821146645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,8192,0.01776426633199056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,2560,0.006704000135262807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,2560,0.01772159934043884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,7168,0.015940266847610473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,2048,0.00591893345117569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,2048,0.017534933487574258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,3584,0.019016534090042114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,1536,0.00526506652434667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,1536,0.016697599490483602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,6144,0.013885866602261862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,1024,0.004444799820582072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,5120,0.012300800283749897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,1024,0.01602773368358612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,4096,0.01053013304869334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,768,0.004092800120512644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,768,0.015622400244077048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,3072,0.0093642661968867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,512,0.003555200000603994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,512,0.015409066279729208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,3584,0.010205866893132527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,256,0.0032629333436489105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,2048,0.007739733159542084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,256,0.01498240033785502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,128,0.003009066730737686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,2560,0.008898133039474487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,128,0.0151936004559199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,64,0.0028490667541821797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,64,0.014620799819628397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,32,0.002935466667016347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,65536,0.042235732078552246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,65536,0.09270186424255371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,1536,0.007225599884986877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,51200,0.0350816011428833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,51200,0.07392640113830566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,32,0.01448319951693217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,16384,0.012583466370900473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,16384,0.02956906755765279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,65536,0.11556373437245686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,12288,0.010648533701896667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,12288,0.02590293288230896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,1024,0.0062389334042867025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,10240,0.009716266393661499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,10240,0.02448106606801351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,768,0.005817600091298421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,8192,0.008731733759244282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,51200,0.09060160319010416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,8192,0.023481599489847817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,7168,0.010913067062695821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,512,0.005312000215053558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,7168,0.022073600689570108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,256,0.005132799843947092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,6144,0.0086218665043513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,128,0.004774400095144907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,6144,0.021802665789922078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,5120,0.008970666925112407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,5120,0.020547199249267577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,4096,0.007923200229803721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,4096,0.019556266069412232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,16384,0.030478932460149127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,3584,0.0076000000039736434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,3584,0.0191103994846344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,12288,0.023668267329533896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,3072,0.007181866466999054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,3072,0.018093866109848023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,3072,0.009268266956011455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,2560,0.006369066735108693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,10240,0.02030186653137207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,2560,0.017638399203618368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,2048,0.0056426664193471275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,8192,0.016839466492335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,2048,0.01686613361040751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,2048,0.007705600063006084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,1536,0.004952533543109894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,1536,0.01648319959640503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,7168,0.01541759967803955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,1024,0.004364799956480662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,6144,0.013521066308021546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,1024,0.015735466281572977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,768,0.003881600002447764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,5120,0.01209173301855723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,512,0.0036138666172822317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,768,0.015410133202870688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,512,0.015150933464368185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,256,0.0032661333680152893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,4096,0.010398933291435241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,256,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,256,0.005027199784914652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,128,0.003017599880695343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,128,0.01476800044377645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,64,0.002916266769170761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,3584,0.010088533163070679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,64,0.01458133359750112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,32,0.0029290666182835894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,32,0.014731733004252115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,65536,0.08525439898173014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,2560,0.008769067128499348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,51200,0.024173865715662636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,51200,0.06666560173034668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,65536,0.029026132822036744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,16384,0.011698133746782939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,16384,0.026572799682617186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,1536,0.007004799942175548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,12288,0.009304533402125042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,1024,0.006162133316198985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,12288,0.025785599152247114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,768,0.005662933488686879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,10240,0.008513066172599792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,10240,0.023998934030532836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,512,0.005351466437180838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,8192,0.007388799885908763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,8192,0.021860265731811525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,128,0.004782933493455251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,7168,0.006989866495132446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,7168,0.02107093334197998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,65536,0.10578560034434001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,6144,0.008460799853007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,6144,0.020463999112447104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,51200,0.08454720179239908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,5120,0.007810133198897044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,5120,0.021206400791803994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,5120,0.012076800068219502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,4096,0.0071370666225751235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,4096,0.019593600432078043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,16384,0.02911679943402608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,3584,0.006757333377997081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,3584,0.019563732544581096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,12288,0.02262506683667501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,3072,0.006417066852251689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,10240,0.019530665874481202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,3072,0.017961599429448447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,3072,0.009197866916656494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,2560,0.00631039987007777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,2560,0.017697066068649292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,2048,0.005499733487764994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,8192,0.016375466187795003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,2048,0.01714986761411031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,1536,0.004836266736189524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,7168,0.015357866883277893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,1536,0.016295466820398966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,1024,0.004145066688458125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,1024,0.01590720017751058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,768,0.0037717332442601522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,768,0.015338666240374246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,6144,0.013406933347384135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,512,0.0035146666069825493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,512,0.015438933173815408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,512,0.005354666709899902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,256,0.0031167998909950255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,256,0.014990933736165366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,4096,0.010580266515413921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,128,0.002940800040960312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,128,0.01515733301639557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,128,0.00468800018231074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,64,0.0028384000062942503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,64,0.014757333199183145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,32,0.0030069333811601003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,32,0.01504533290863037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,65536,0.023219199975331624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,65536,0.08130239645640056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,3584,0.010036266843477885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,51200,0.01781546672185262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,65536,0.10762879848480225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,51200,0.06231360038121542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,51200,0.08584213256835938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,16384,0.008005333443482716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,16384,0.026225066184997557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,16384,0.02878613273302714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,12288,0.008117333551247915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,12288,0.02286613384882609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,10240,0.0072736000021298725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,10240,0.02319999933242798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,10240,0.019684267044067384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,8192,0.00673280010620753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,8192,0.021387734015782676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,2560,0.008733866612116496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,2048,0.007692799965540569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,7168,0.006411733229955037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,1536,0.006872533261775971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,7168,0.015378133455912272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,7168,0.02109760046005249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,6144,0.0062165334820747375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,5120,0.006584533552328746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,6144,0.013398399949073792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,6144,0.020216532548268638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,4096,0.006188799937566122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,1024,0.005949866771697998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,5120,0.02016426722208659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,3584,0.00660693347454071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,4096,0.020083200931549073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,3584,0.01858560045560201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,3072,0.006263466676076253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,768,0.005627733469009399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,3072,0.01795413295427958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,3072,0.009019733468691508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,2560,0.006033066908518473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,2560,0.017449599504470826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,12288,0.023144533236821495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,2560,0.008582400282224019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,2048,0.005343999962011973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,1536,0.004693333307902018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,2048,0.0074986666440963745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,2048,0.016937599579493205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,1024,0.004123733441034953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,1536,0.01622293293476105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,1024,0.015770666797955833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,768,0.0037802666425704955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,768,0.015475199619928996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,768,0.0056981335083643595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,512,0.0034175999462604523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,256,0.00487360010544459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,512,0.015250133474667868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,8192,0.016545066237449647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,256,0.003058133274316788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,256,0.014778666694959006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,5120,0.011948800086975098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,4096,0.010354133447011311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,128,0.0028394666810830434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,3584,0.009917866190274556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,128,0.004818133513132731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,128,0.014492799838383993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,64,0.0027477333943049112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,32,0.00276053324341774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,64,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,32,0.014703999956448874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,65536,0.016910932461420693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,65536,0.07888320287068686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,51200,0.05940586725870768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,16384,0.006983466446399689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,1536,0.0070165331164995836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,12288,0.006345599889755249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,16384,0.02537386616071065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,10240,0.006483200192451477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,12288,0.022959999243418374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,8192,0.006354133288065593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,10240,0.0225600004196167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,7168,0.006258133550484974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,8192,0.02101759910583496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,1024,0.006075733403364817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,6144,0.0060928001999855045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,7168,0.020870399475097657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,5120,0.0064085334539413456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,4096,0.006067200005054474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,6144,0.02034239967664083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,5120,0.020080000162124634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,3584,0.006516266862551372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,3072,0.00621973325808843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,4096,0.018633600076039633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,3584,0.01846933364868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,2560,0.006121600170930227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,3072,0.018228266636530557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,2048,0.005329066514968872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,2560,0.017207467555999757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,1536,0.004676266511281332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,2048,0.017154133319854735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,512,0.005353599786758423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,1024,0.003945599993069967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,1536,0.016343466440836587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,768,0.0036746665835380556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,512,0.0036373332142829893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,256,0.0029866665601730345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,768,0.015603199601173401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,128,0.0028064000109831494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,512,0.015159466862678527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,64,0.002701866626739502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,256,0.015175466736157736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,65536,0.013059199849764506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,32,0.014566399653752646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,51200,0.010146133104960124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,65536,0.07722986539204915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,51200,0.05743786493937174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,16384,0.006244266529877981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,12288,0.006232533355553945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,16384,0.025468800465265912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,10240,0.006385066608587901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,12288,0.023057067394256593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,8192,0.006355200211207073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,10240,0.02313813368479411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,7168,0.006167466441790262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,8192,0.02087679902712504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,6144,0.005966933568318685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,51200,0.00974079966545105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,7168,0.020957867304484047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,5120,0.006348800162474315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,6144,0.020065067211786906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,5120,0.020105600357055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,4096,0.005913599828879039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,3584,0.006431999802589417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,4096,0.018927999337514243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,3584,0.01843199928601583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,2560,0.0060149331887563075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,3072,0.017655466000239053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,2048,0.005427200098832448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,2560,0.01794559955596924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,1536,0.004709333181381226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,2048,0.017409066359202065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,1024,0.003994666785001755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,1536,0.016523733735084534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,768,0.0037098666032155357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,1024,0.015763200322786965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,512,0.003352533280849457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,768,0.016668800512949625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,256,0.003052799900372823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,1024,0.01619733373324076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,128,0.0028266665836175283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,128,0.014419200023015341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,128,0.014760532975196838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,64,0.0026954665780067446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,64,0.014355199535687766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,32,0.002630399912595749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,64,0.014615466197331747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,32,0.014432000120480857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,256,0.004990933338801066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,3072,0.006098133325576782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,512,0.015100799997647605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,256,0.014802133043607077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,32,0.00264533335963885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,16384,0.7913514455159505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,16384,0.42984533309936523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,12288,0.606167475382487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,51200,1.2873140970865884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,12288,0.33597440719604493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,12288,0.4783381462097168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,10240,0.5064874649047851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,10240,0.32434558868408203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,10240,0.399019718170166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,51200,2.503815460205078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,8192,0.23487253189086915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,8192,0.4049248059590657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,7168,0.36179946263631185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,7168,0.27618239720662435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,6144,0.18572160402933757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,6144,0.3139509201049805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,7168,0.23728747367858888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,5120,0.1588469346364339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,5120,0.26328746477762854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,4096,0.2430965264638265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,4096,0.13219839731852215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,3584,0.12000853220621746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,3584,0.21539306640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,3584,0.13552640279134115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,3072,0.18724053700764973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,3072,0.1062165339787801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,3072,0.11741440296173096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,2560,0.14194879531860352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,2560,0.09260053634643554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,2048,0.11656320095062256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,2048,0.07893866697947184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,2048,0.08102826277414957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,1536,0.09201386769612631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,1536,0.06591360171635946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,1536,0.06645973523457846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,1024,0.06454613208770751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,1024,0.0530730684598287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,51200,2.1236095428466797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,768,0.061203201611836754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,768,0.051736533641815186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,5120,0.20023253758748374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,16384,0.6615626653035481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,512,0.044164268175760905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,512,0.043841067949930826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,256,0.027589333057403565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,256,0.03358826637268066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,256,0.032026666402816775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,128,0.01955839991569519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,128,0.02911679943402608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,2560,0.10126612981160481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,64,0.014966400464375815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,64,0.028354134162267047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,65536,32,0.015989333391189575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,65536,32,0.03123733401298523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,8192,0.31581013997395835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,6144,0.23872000376383462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,4096,0.15679465929667155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,65536,1.3593429565429687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,1024,0.04989440043767293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,51200,2.1091232299804688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,65536,2.696942901611328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,16384,0.7285066604614258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,128,0.02799786726633708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,16384,0.39521706899007164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,51200,1.0724906921386719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,12288,0.48945172627766925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,12288,0.3058368047078451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,512,0.036099199453989664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,10240,0.4152202606201172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,10240,0.23235947291056314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,8192,0.1907434622446696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,65536,768,0.04629546801249186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,8192,0.3337194760640462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,7168,0.17763946851094564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,65536,2.1908543904622393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,6144,0.2910090764363607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,7168,0.29111146926879883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,6144,0.15282452901204427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,5120,0.21286932627360025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,16384,0.535144551595052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,5120,0.1311957359313965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,51200,1.7213194529215496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,12288,0.4223306655883789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,5120,0.15991573333740233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,4096,0.17276159922281903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,4096,0.12180266380310059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,3584,0.15525546073913574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,3584,0.10071039994557698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,10240,0.32852586110432946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,3072,0.1361024061838786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,3584,0.10723093350728352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,2560,0.11491200129191081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,2560,0.07848853270212809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,2048,0.09454507033030192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,2048,0.0674944003423055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,8192,0.2672959963480631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,7168,0.23391253153483071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,1536,0.08204373518625895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,1536,0.057100800673166904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,3072,0.08907306989034017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,1024,0.058856534957885745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,1024,0.04670613209406535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,1024,0.04377919832865397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,768,0.043381333351135254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,768,0.041934935251871745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,6144,0.20154986381530762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,512,0.037481598059336346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,512,0.036505599816640213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,4096,0.13210026423136395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,256,0.025380265712738038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,256,0.031096533934275312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,3072,0.09596266746520996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,128,0.017939200003941856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,128,0.02856106758117676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,2560,0.07969600359598795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,64,0.016295466820398966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,64,0.028382933139801024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,2048,0.06543360153834024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,51200,32,0.01523413360118866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,51200,32,0.02755519946416219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,1536,0.04911786715189616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,65536,0.456112003326416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,65536,0.7831381479899089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,768,0.03300159970919291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,51200,0.36013654073079426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,51200,0.616045888264974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,16384,0.2025983969370524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,16384,0.13058026631673175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,12288,0.1569482644399007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,512,0.02722346584002177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,51200,0.5074837366739909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,16384,0.16465813318888348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,12288,0.10340053240458172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,10240,0.12998507022857667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,12288,0.12311253547668458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,8192,0.1053440014521281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,10240,0.10117333730061848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,8192,0.07474453449249267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,7168,0.09309226671854655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,7168,0.07095572948455811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,6144,0.0818015972773234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,6144,0.06374826828638712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,6144,0.06363626718521118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,5120,0.07057173252105713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,5120,0.053564798831939694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,5120,0.05372053384780884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,4096,0.05695039828618368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,128,0.018282665808995565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,4096,0.04761813481648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,3584,0.05029439926147461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,3584,0.04370880126953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,3072,0.045049599806467694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,3072,0.0395306666692098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,7168,0.07157759666442871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,2560,0.03852800130844116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,51200,256,0.021528534094492593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,2560,0.035957332452138266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,2560,0.030195200443267824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,2048,0.032203733921051025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,2048,0.03146026730537414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,1536,0.025702399015426636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,1536,0.02700693408648173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,10240,0.10173546473185222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,65536,0.6439082463582356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,1024,0.020282665888468422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,1024,0.016685867309570314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,1024,0.026687999566396076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,768,0.014297599593798319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,768,0.02455679972966512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,512,0.011724799871444702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,512,0.021255467335383096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,512,0.014172800381978354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,256,0.008231466511885326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,256,0.018347734212875368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,4096,0.04381866852442424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,128,0.0059349333246548975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,128,0.01716266671816508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,8192,0.08432426452636718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,2048,0.025450666745503742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,64,0.005353599786758423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,16384,32,0.005774933099746704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,64,0.01774079998334249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,16384,32,0.018114133675893148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,3584,0.03784746726353963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,1536,0.021010132630666097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,65536,0.3600213368733724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,65536,0.5789130528767903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,65536,0.5564213434855143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,51200,0.4567925453186035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,16384,0.1510335922241211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,51200,0.2810389200846354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,16384,0.11866026719411213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,12288,0.11949333349863689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,12288,0.08386346499125162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,12288,0.10874880154927571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,10240,0.09731840292612712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,10240,0.07294507026672363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,3072,0.03333333333333333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,8192,0.07926613489786784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,8192,0.06106346845626831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,256,0.010626133282979329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,7168,0.07031893730163574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,7168,0.055668266614278164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,128,0.009852799773216247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,51200,0.4391018549601237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,6144,0.06172266801198324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,6144,0.05522666772206625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,5120,0.05275413195292154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,5120,0.045637333393096925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,10240,0.09082240263621012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,4096,0.04324160019556682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,4096,0.039740800857543945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,4096,0.03832533359527588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,3584,0.03887253204981486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,3584,0.03714239994684855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,3072,0.034468265374501546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,8192,0.0728991985321045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,16384,768,0.01511573294798533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,3072,0.034202667077382405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,2560,0.029665066798528032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,2560,0.030573866764704388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,2048,0.02485760052998861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,2048,0.027082665761311846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,16384,0.14363199869791668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,1536,0.021916800737380983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,1536,0.024086399873097738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,7168,0.06457813183466593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,1024,0.013672533631324767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,1024,0.021641600131988525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,6144,0.056201601028442384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,768,0.011286399761835734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,768,0.02063680092493693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,5120,0.04848320086797078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,512,0.009174399574597676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,512,0.018734933932622273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,3584,0.03428373336791992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,256,0.006274133423964183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,3072,0.028838400046030683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,256,0.008552533388137818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,256,0.017222400506337485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,128,0.005014400184154511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,128,0.01728106737136841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,64,0.005111466844876607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,2560,0.02459519902865092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,64,0.01717546582221985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,2048,0.02076693375905355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,12288,32,0.005275733272234599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,12288,32,0.017416532834370932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,1536,0.017358932892481485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,65536,0.3420650800069173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,65536,0.5377493540445963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,51200,0.4253418604532878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,1024,0.014412800470987955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,16384,0.1374666690826416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,51200,0.24750399589538574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,16384,0.09382719993591308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,12288,0.10063040256500244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,768,0.012698666254679362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,10240,0.08310080369313558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,10240,0.06437653303146362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,512,0.010938666264216105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,8192,0.06829226811726888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,12288,0.07572267055511475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,8192,0.05540800094604492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,7168,0.06042346556981405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,12288,128,0.007877333462238312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,7168,0.05123840173085531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,7168,0.05998293161392212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,6144,0.0532970666885376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,6144,0.04729280074437459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,5120,0.0452565352121989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,5120,0.04156266848246257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,65536,0.5205866813659668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,4096,0.042056532700856526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,4096,0.03694613377253215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,4096,0.03593920071919759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,3584,0.03402666648228963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,16384,0.1330357313156128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,51200,0.40709759394327805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,3584,0.03449386755625407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,3072,0.0336575984954834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,3072,0.03155946731567383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,12288,0.09627839724222818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,2560,0.026961066325505573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,2560,0.02849919994672139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,2048,0.022258132696151733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,2048,0.025389866034189863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,10240,0.08214399814605713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,1536,0.017925333976745606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,1536,0.023386667172114052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,8192,0.06848106384277344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,1024,0.013205333550771078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,1024,0.02100480000178019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,6144,0.05165546735127767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,768,0.011065600315729777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,5120,0.04412480195363362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,768,0.020117332537968956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,3584,0.030719999472300214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,3072,0.027577600876490277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,512,0.008725333213806152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,512,0.017591466506322227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,256,0.006342400113741558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,2560,0.024256000916163124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,256,0.017139200369517008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,128,0.005037866532802582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,128,0.016702934106191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,2048,0.019690666596094766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,64,0.004470400015513102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,64,0.016371200482050575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,10240,32,0.004542933404445648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,10240,32,0.01644159952799479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,1536,0.016101333498954772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,65536,0.2754410743713379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,65536,0.41122881571451825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,1024,0.013347199559211731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,768,0.011657599608103435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,51200,0.20729279518127441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,51200,0.329203192392985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,16384,0.1027616024017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,16384,0.07978666623433431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,12288,0.08034026622772217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,16384,0.0852224032084147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,51200,0.25971733729044594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,12288,0.06414186557133993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,10240,0.06734506289164224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,10240,0.05346986850102743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,8192,0.05436266660690307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,10240,0.05574186642964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,8192,0.04751360019048055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,7168,0.04832853476206462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,7168,0.044536534945170084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,6144,0.04292693138122559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,6144,0.03974506855010986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,512,0.00918506681919098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,5120,0.03656320174535115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,256,0.007659733295440674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,5120,0.03565973440806071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,4096,0.03020693262418111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,10240,128,0.007098666826883952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,4096,0.03144426743189494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,3584,0.02725013295809428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,3584,0.029190399249394733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,65536,0.3388789176940918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,3072,0.027075199286142985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,12288,0.06381973425547281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,3072,0.026586665709813433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,2560,0.020871466398239134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,2560,0.024689066410064697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,7168,0.04015680154164632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,8192,0.04474453528722127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,2048,0.01689280072848002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,2048,0.015602133671442666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,2048,0.023228800296783446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,1536,0.013222400347391763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,1536,0.02141759991645813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,6144,0.03481173515319824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,1024,0.009967999656995137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,1024,0.019589332739512126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,768,0.008622933427492778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,768,0.017961599429448447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,768,0.01023573378721873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,512,0.007067733506361644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,512,0.017166932423909508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,5120,0.031243733565012616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,256,0.004958933095137278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,4096,0.02472426692644755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,256,0.016680532693862916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,128,0.004502399762471517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,128,0.015893333156903586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,3584,0.022196267048517862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,64,0.003928533444801966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,64,0.016265599926312765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,8192,32,0.004433066646258036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,8192,32,0.016611199577649435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,65536,0.3846101442972819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,3072,0.019933867454528808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,65536,0.291429328918457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,2560,0.01748159925142924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,51200,0.21748800277709962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,51200,0.3974805196126302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,1536,0.013123200337092081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,16384,0.11565866470336914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,16384,0.07795306841532389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,1024,0.011224533120791119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,12288,0.07814613183339437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,12288,0.06128426790237427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,10240,0.07090240319569906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,512,0.008450133601824443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,10240,0.05402026573816935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,8192,0.054960000514984134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,8192,0.04592853387196859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,8192,0.04163839817047119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,256,0.007375999788443248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,7168,0.04953706661860148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,7168,0.04287999868392944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,7168,0.03715840180714925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,6144,0.042565333843231204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,8192,128,0.006989866495132446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,6144,0.038626134395599365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,5120,0.036185598373413085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,5120,0.03502293427785237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,65536,0.3128767967224121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,4096,0.0310698668162028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,4096,0.03183146715164185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,4096,0.023563732703526817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,3584,0.027589333057403565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,3584,0.02898240089416504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,51200,0.2516842683156332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,3584,0.021367466449737547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,3072,0.023661865790685018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,3072,0.027445334196090698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,16384,0.08301226298014322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,2560,0.02040533423423767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,2560,0.024825600783030193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,2560,0.016748799880345663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,2048,0.017066667477289833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,2048,0.023370667298634847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,1536,0.013321600357691445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,10240,0.05056426525115967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,1536,0.021052799622217813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,1024,0.010248532891273499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,1024,0.01916586756706238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,6144,0.03337599833806355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,768,0.008142933249473572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,768,0.017754666010538735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,512,0.006693333387374878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,512,0.017130666971206666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,5120,0.027754666407903035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,256,0.004863999783992767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,256,0.01656426688035329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,12288,0.05894506772359213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,128,0.004204800228277842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,128,0.016153599818547568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,768,0.00874773363272349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,64,0.004265599946180979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,64,0.01620586713155111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,7168,32,0.004342400034268697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,7168,32,0.016289066274960837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,3072,0.019317332903544107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,65536,0.33367465337117513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,2048,0.014199466506640116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,65536,0.25286293029785156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,1536,0.012141866485277812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,1024,0.01039573351542155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,51200,0.267739741007487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,51200,0.20009919802347817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,16384,0.10273280143737792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,16384,0.0748085339864095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,12288,0.07647999922434488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,12288,0.058956801891326904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,10240,0.06459519863128663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,10240,0.05220693349838257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,512,0.0076000000039736434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,8192,0.050919465223948156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,8192,0.044410665829976395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,8192,0.040428801377614336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,7168,0.04590719938278198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,256,0.006444799900054932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,7168,0.04044373432795207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,7168,128,0.006025599936644236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,6144,0.0390720009803772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,6144,0.03115306695302327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,5120,0.03340693314870198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,5120,0.03387519915898641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,6144,0.03689599831899007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,4096,0.027905066808064777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,4096,0.029226666688919066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,4096,0.022217599550882976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,3584,0.024630399545033772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,3584,0.027566933631896974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,51200,0.22932373682657875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,65536,0.28898134231567385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,16384,0.07589013576507568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,3072,0.021995733181635536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,2560,0.018971733252207437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,3072,0.025840000311533613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,12288,0.05531626542409261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,2560,0.024078933397928874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,2560,0.016301866372426352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,2048,0.015821866194407144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,2048,0.022869332631429037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,1536,0.020801067352294922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,10240,0.04893333514531453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,1024,0.009453866879145305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,1024,0.018054399887720743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,7168,0.034764798482259114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,768,0.007709866762161255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,768,0.017144532998402913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,5120,0.027506132920583088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,512,0.006133333345254262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,512,0.01694186727205912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,1536,0.012545067071914672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,256,0.004692266881465912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,256,0.015961600343386333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,3584,0.01967039903004964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,3072,0.01802560091018677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,128,0.004049066702524821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,128,0.0162581334511439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,2048,0.013643733660380044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,64,0.004048000027736028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,1536,0.01169599990049998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,64,0.01623253325621287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,6144,32,0.00411520004272461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,6144,32,0.016424533724784852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,1024,0.009572266538937887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,65536,0.271283213297526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,65536,0.20652052561442055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,51200,0.16268906593322754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,65536,0.27172374725341797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,768,0.008588799834251403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,16384,0.06288533210754395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,512,0.007523199915885926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,12288,0.05943573315938314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,256,0.006435200075308482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,12288,0.05058666865030924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,10240,0.045324798425038657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,10240,0.045943466822306316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,51200,0.2163914680480957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,8192,0.04164586861928304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,16384,0.07606399854024251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,8192,0.0394538680712382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,7168,0.03606826861699422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,7168,0.036161065101623535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,6144,0.03148373365402222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,6144,128,0.006040533383687338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,6144,0.033037867148717245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,5120,0.028446932633717854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,5120,0.030086400111516316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,51200,0.2123583952585856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,4096,0.02330026626586914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,4096,0.026532266537348432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,16384,0.06992746988932291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,4096,0.020773333311080933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,12288,0.05136426687240601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,3584,0.01885226567586263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,3584,0.025230934222539265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,3584,0.018462934096654258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,3072,0.0165994664033254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,2560,0.01543786625067393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,3072,0.024537599086761473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,2560,0.022846933205922446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,2048,0.013386666774749756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,10240,0.0443338672320048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,8192,0.037003731727600096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,1536,0.009547733267148336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,1536,0.01993173360824585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,7168,0.032467200358708696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,1024,0.007793066898981731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,1024,0.01726186672846476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,6144,0.028703999519348145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,1024,0.008925867080688477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,768,0.006867200136184692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,768,0.017078399658203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,768,0.007956266899903615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,512,0.005670399963855743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,512,0.016770132382710776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,5120,0.02462079922358195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,2048,0.021293866634368896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,256,0.01611306667327881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,128,0.0039264000952243805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,128,0.01542080044746399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,128,0.0054837331175804135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,64,0.0035946667194366455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,64,0.015474133690198264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,32,0.003689600030581156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,5120,32,0.0156960000594457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,65536,0.21840106646219887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,65536,0.17789333661397297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,5120,256,0.004317866762479147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,3072,0.016473600268363954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,2560,0.015025066335995993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,51200,0.17433600425720214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,51200,0.13812373479207357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,16384,0.06270186503728231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,2048,0.012449066837628682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,16384,0.060819200674692785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,12288,0.050020265579223636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,12288,0.05534186760584513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,10240,0.04628586769104004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,10240,0.04310293197631836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,1536,0.010680533448855082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,8192,0.039478401343027755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,8192,0.0377781351407369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,512,0.0071722666422526045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,7168,0.03422293265660604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,7168,0.033931732177734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,5120,256,0.0059008002281188965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,6144,0.029822933673858642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,6144,0.03099946578343709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,6144,0.026503467559814455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,5120,0.02499413291613261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,5120,0.028593067328135175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,65536,0.2437877337137858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,4096,0.020833067099253335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,16384,0.06256106694539389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,51200,0.19262827237447103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,12288,0.04659519990285237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,3584,0.019180800517400107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,3584,0.024947200218836466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,10240,0.04061226844787598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,3072,0.016863999764124553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,3072,0.023432532946268715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,8192,0.033829331398010254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,7168,0.03025706609090169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,2560,0.014869333306948344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,2560,0.014076800147692362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,2560,0.021911466121673585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,2048,0.01234879990418752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,2048,0.02085439960161845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,1536,0.009750399986902874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,1536,0.018605866034825645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,5120,0.02288960019747416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,1024,0.007781333227952321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,1024,0.017709867159525553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,1024,0.008613333106040955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,768,0.006332799792289734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,768,0.016827734311421712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,4096,0.026121600468953447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,512,0.005264000097910563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,512,0.01643946667512258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,4096,0.019217065970102944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,256,0.0041696002086003625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,256,0.0155349334081014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,128,0.0037077332536379496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,128,0.01524906655152639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,3584,0.017707733313242595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,64,0.0033791999022165934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,64,0.015361066659291586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,3072,0.01585919956366221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,4096,32,0.0035904000202814737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,4096,32,0.01576640009880066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,65536,0.18917333285013835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,65536,0.16582293510437013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,51200,0.15370346705118815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,51200,0.12950293223063153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,2048,0.012045866250991822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,16384,0.05861546595891317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,1536,0.010507733623186747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,16384,0.052400000890096034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,12288,0.046402132511138915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,12288,0.042650667826334636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,768,0.008218666911125183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,10240,0.034517331918080645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,10240,0.03822186787923177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,512,0.007246933380762736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,8192,0.0279530664285024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,8192,0.03691093524297078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,256,0.006237866481145223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,7168,0.025491199890772503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,7168,0.0341536005338033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,4096,128,0.005895466605822245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,7168,0.029677865902582805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,6144,0.022753065824508666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,6144,0.03123093247413635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,5120,0.019543466965357463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,5120,0.028652799129486085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,65536,0.22836267153422035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,4096,0.020848000049591066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,4096,0.025639466444651288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,51200,0.182585604985555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,3584,0.018589866161346436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,16384,0.06103893518447876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,3584,0.024962133169174193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,3072,0.01635840038458506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,3072,0.023112533489863078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,12288,0.045525332291920975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,2560,0.013527466853459676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,2560,0.021705599625905354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,10240,0.03926186561584473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,2048,0.012072533369064331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,8192,0.03299199938774109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,2048,0.020637865861256918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,1536,0.00960106650988261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,1536,0.018338133891423546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,6144,0.025601067145665485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,1024,0.007659733295440674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,1024,0.017324799299240114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,5120,0.022781866788864135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,768,0.005797333518664042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,768,0.01663146714369456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,4096,0.01816213329633077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,512,0.004841599861780802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,512,0.015845333536465965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,2048,0.011126400033632914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,256,0.0038880000511805216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,256,0.015638400117556253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,3584,0.01630400021870931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,128,0.003475199888149897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,128,0.015201066931088766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,3072,0.01502826710542043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,128,0.0055167997876803074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,64,0.0034453332424163817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3584,32,0.0034741332133611047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,64,0.015338666240374246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3584,32,0.015579733252525329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,2560,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,65536,0.1715167999267578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,65536,0.15019200642903646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,51200,0.13744319279988607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,51200,0.12548480033874512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,1536,0.009504000345865887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,16384,0.049728000164031984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,16384,0.05652053356170654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,1024,0.00840106705824534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,12288,0.05021013418833414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,768,0.007396266857783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,12288,0.045414400100708005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,10240,0.040931200981140135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,10240,0.040504535039265946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,512,0.006677333513895671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,8192,0.03405760129292806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,8192,0.03458880186080933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3584,256,0.0058112000425656635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,7168,0.03190933267275493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,7168,0.03219306667645772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,65536,0.21882987022399902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,6144,0.02658560077349345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,6144,0.029666133721669513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,5120,0.022394667069117226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,5120,0.02680213252703349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,51200,0.18146665891011554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,4096,0.018412800629933675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,4096,0.024078933397928874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,12288,0.044462935129801436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,16384,0.05819199879964193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,3584,0.017362133661905924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,3584,0.023538132508595787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,3072,0.014937600493431092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,3072,0.022423466046651207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,10240,0.037539199988047285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,2560,0.012769066294034324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,2560,0.020966400702794395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,8192,0.031113600730895995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,2048,0.011377066373825073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,2048,0.01973653237024943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,7168,0.02722773353258769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,1536,0.009131733576456707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,1536,0.018242132663726807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,6144,0.024936532974243163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,1024,0.0064181332786877945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,1024,0.01720213294029236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,5120,0.021631999810536703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,768,0.0054666668176651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,768,0.016105600198109946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,4096,0.017866667111714682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,512,0.0047199999292691554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,512,0.016025599837303162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,3584,0.015864533185958863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,256,0.0037600000699361167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,256,0.015560533603032431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,3072,0.014264532923698425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,128,0.003706666578849157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,2560,0.012989866733551025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,128,0.015312000115712484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,64,0.0034058667719364167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,64,0.015348266561826071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,3072,32,0.0034944000343481696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,3072,32,0.015542399883270264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,2048,0.010590933760007222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,65536,0.14954346021016438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,65536,0.1380298614501953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,1536,0.008909866213798523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,51200,0.11379093329111736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,51200,0.10994986693064372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,16384,0.044719998041788736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,16384,0.046649599075317384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,1024,0.007980800171693166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,12288,0.0370901346206665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,768,0.0069472000002861025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,12288,0.03857386509577433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,10240,0.03033706744511922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,10240,0.03383466800053914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,512,0.0063498665889104204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,8192,0.025576533873875935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,256,0.005500799914201101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,8192,0.03130879998207092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,8192,0.02900159955024719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,7168,0.022680532932281495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,7168,0.02722240090370178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,6144,0.01943146586418152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,3072,128,0.005131733417510986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,6144,0.025425066550572712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,6144,0.023780266443888344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,5120,0.014535466829935709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,5120,0.024218666553497314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,4096,0.012032000223795573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,4096,0.022445867458979286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,65536,0.2095871925354004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,51200,0.16622826258341472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,3584,0.011116799712181092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,3584,0.02137813369433085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,3072,0.012044800321261089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,3072,0.02060479919115702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,16384,0.05533013343811035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,2560,0.008964266379674275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,12288,0.04209386507670085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,2560,0.019697066148122153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,10240,0.036025599638621016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,2048,0.009221333265304565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,2048,0.01769919991493225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,7168,0.026257065931955974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,1536,0.006814933319886525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,1536,0.01723519961039225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,5120,0.02023786703745524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,1024,0.005705599983533224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,1024,0.016618667046229045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,4096,0.01665173371632894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,768,0.0049898669123649595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,768,0.01650879979133606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,3584,0.014885333180427552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,3072,0.013591466347376504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,512,0.004175999760627746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,512,0.006326400240262349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,256,0.00383786658445994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,256,0.015495466192563376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,128,0.0034634667138258614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,128,0.01493013302485148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,2560,0.011959466338157653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,64,0.0031541332602500914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,64,0.014958932995796204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2560,32,0.0031957333286603295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,32,0.015075199802716575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,65536,0.11463359991709392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,2048,0.01032319962978363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,65536,0.12394026915232341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2560,512,0.01613759994506836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,51200,0.09292373657226563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,51200,0.0977194627126058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,1536,0.008794666330019633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,16384,0.03900800148646037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,16384,0.043594666322072345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,12288,0.03157866597175598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,12288,0.03545279900232951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,1024,0.00766293356815974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,10240,0.029307732979456585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,768,0.006958933174610138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,256,0.005522133409976959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,8192,0.025628799200057985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,8192,0.02744320034980774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2560,128,0.005266133447488149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,10240,0.030266666412353517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,7168,0.022859734296798707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,7168,0.025914667050043742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,6144,0.023385600248972575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,6144,0.023818665742874147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,65536,0.20040426254272461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,5120,0.01728640000025431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,5120,0.022641066710154215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,51200,0.15489706993103028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,16384,0.05121279954910278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,4096,0.016452266772588094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,4096,0.021945599714914957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,3584,0.013446399569511413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,12288,0.039795200030008956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,3584,0.020730666319529214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,3072,0.013184000054995218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,3072,0.019308799505233766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,3072,0.013314132889111837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,2560,0.011187199751536052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,10240,0.03391146659851074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,2560,0.01833066741625468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,2048,0.009048533439636231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,2048,0.017356799046198527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,7168,0.02491413354873657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,8192,0.028386133909225463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,1536,0.00710399995247523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,1536,0.017922133207321167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,1024,0.005456000069777171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,6144,0.02267520030339559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,5120,0.020066134134928384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,768,0.0047872001926104225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,768,0.016611199577649435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,4096,0.016056533654530844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,512,0.004062933226426443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,512,0.01601920028527578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,3584,0.015507200360298156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,256,0.0038474666575590765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,256,0.015310933192571005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,1024,0.017092265685399375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,128,0.0034005333979924522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,128,0.015174399813016257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,2048,0.00976213316122691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,64,0.0032320000231266023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,64,0.015076266725858054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,2048,32,0.003335466732581457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,2048,32,0.015041066209475198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,65536,0.08973973592122396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,65536,0.11003946463267009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,2560,0.01169599990049998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,1536,0.008582400282224019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,51200,0.0726154645284017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,51200,0.0874229351679484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,16384,0.03579306602478027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,1024,0.007369600236415863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,16384,0.04050133228302002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,768,0.006759466727574666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,12288,0.028252800305684406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,512,0.00629013329744339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,12288,0.03253013292948405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,10240,0.02773653268814087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,256,0.005526400109132131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,10240,0.029399466514587403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,8192,0.016254933675130208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,8192,0.02627519965171814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,2048,128,0.005070933202902476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,7168,0.02039360006650289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,7168,0.02590293288230896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,65536,0.18791786829630536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,6144,0.014521599809328715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,6144,0.023487999041875204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,51200,0.15182506243387858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,5120,0.012055466572443645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,16384,0.04974826574325562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,5120,0.02236479918162028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,4096,0.009998933474222819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,4096,0.021149865786234536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,3584,0.010082133611043294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,3584,0.01990506649017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,12288,0.03708266814549764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,3072,0.008585600058237712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,3072,0.018886399269104005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,8192,0.02718399961789449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,2560,0.009171199798583985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,2560,0.01858133276303609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,7168,0.0236842672030131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,2048,0.007501866420110066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,2048,0.017768534024556477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,10240,0.031754666566848756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,1536,0.006100266675154368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,1536,0.016990933815638223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,6144,0.021780266364415487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,1024,0.004955733319123586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,1024,0.016214399536450704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,5120,0.01771413286526998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,768,0.004420266548792521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,768,0.016215466459592185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,4096,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,512,0.004012800008058548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,512,0.015783466895421348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,3584,0.013667200009028116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,256,0.0035103999078273775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,256,0.015337600310643514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,3072,0.011901866396268208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,128,0.0032138665517171226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,128,0.015460266669591268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,2560,0.01087679962317149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,64,0.002977066735426585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,64,0.014841600259145101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1536,32,0.003035733352104823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1536,32,0.014965333541234336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,65536,0.06657386620839437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,65536,0.09632639884948731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,2048,0.009244799613952637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,51200,0.05467733144760132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,1536,0.008071466783682505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,51200,0.07650880018870035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,16384,0.02221333384513855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,1024,0.007361066838105519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,16384,0.03381866614023844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,12288,0.017755732933680216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,12288,0.032025599479675294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,768,0.006628266473611195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,10240,0.021891200542449953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,512,0.006110933423042297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,10240,0.02991360028584798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,8192,0.0178272008895874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,256,0.005355733136336008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,8192,0.026972800493240356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,7168,0.01924053430557251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,7168,0.02571626702944438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1536,128,0.005017599960168203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,6144,0.016167466839154564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,6144,0.023705599705378215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,65536,0.17255573272705077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,5120,0.012210133671760558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,5120,0.02177066604296366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,51200,0.135753599802653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,16384,0.04563519954681396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,4096,0.010554666320482891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,4096,0.02028053402900696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,3584,0.010762666662534077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,3584,0.01953493356704712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,12288,0.035521066188812254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,3072,0.008822400371233623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,3072,0.019001599152882895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,10240,0.029659734169642134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,2560,0.007879466811815897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,8192,0.025908267498016356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,2560,0.01869759956995646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,2048,0.006809600194295247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,2048,0.01771946748097738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,7168,0.023090134064356484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,1536,0.00644053320089976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,1536,0.016764799753824867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,6144,0.020230400562286376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,5120,0.017563732465108235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,4096,0.01463573376337687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,1024,0.004910933474699656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,1024,0.016403200229008992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,768,0.004305066665013632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,768,0.01600213348865509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,3584,0.013369599978129068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,512,0.003869866579771042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,512,0.01523413360118866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,3072,0.011818666259447734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,256,0.0033429334561030067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,256,0.015082666277885437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,2560,0.010803199807802836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,128,0.0031541332602500914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,2048,0.008994133273760477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,128,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,64,0.0029994666576385496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,64,0.014681599537531533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,1024,32,0.003031466652949651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,1536,0.007986133297284443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,1024,32,0.014707199732462563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,65536,0.056576001644134524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,65536,0.09089600245157878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,51200,0.047516798973083495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,51200,0.07133119901021322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,16384,0.017900800704956053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,16384,0.03259306748708089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,1024,0.007338666419188182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,12288,0.014971733093261719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,12288,0.0318015992641449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,768,0.006333866715431213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,10240,0.013619200388590495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,512,0.005849599838256836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,10240,0.029047467311223346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,8192,0.014459733168284097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,8192,0.02643520037333171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,256,0.005144533514976501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,7168,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,7168,0.02421440084775289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,1024,128,0.004819199939568837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,6144,0.015224533279736838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,6144,0.023396267493565878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,65536,0.1390218734741211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,5120,0.014879999558130899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,5120,0.021708800395329794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,51200,0.11057173411051432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,4096,0.011678933103879293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,16384,0.03631466627120972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,4096,0.020965333779652914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,3584,0.009924266735712688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,3584,0.01930560072263082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,12288,0.028169600168863933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,3072,0.009111467003822326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,10240,0.02418773372968038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,3072,0.018968532482782997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,2560,0.0083146666487058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,8192,0.020822399854660036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,2560,0.01858666737874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,2048,0.006989866495132446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,2048,0.017858133713404337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,7168,0.017909334103266398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,1536,0.005957333246866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,1536,0.016889599959055583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,6144,0.016335999965667723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,1536,0.007406933108965556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,1024,0.01593493322531382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,5120,0.013724799950917563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,768,0.00451093316078186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,768,0.016014933586120605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,4096,0.011962667107582092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,512,0.0038250667353471124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,512,0.0152319997549057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,3584,0.011392000317573547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,256,0.0033013333876927694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,256,0.014991999665896098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,256,0.005047466854254405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,128,0.0030591999491055804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,128,0.014683733383814493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,1024,0.004763733347256978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,64,0.002918400118748347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,64,0.014806399742762247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,768,32,0.0029578665892283124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,768,32,0.014972800016403198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,65536,0.0405290683110555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,65536,0.08324586550394694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,3072,0.0100490669409434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,51200,0.033633065223693845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,2560,0.009261866410573322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,51200,0.06673386891682943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,16384,0.013945600390434265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,16384,0.030795733133951824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,2048,0.008096000055472057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,12288,0.011767466862996418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,16384,0.031625600655873616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,12288,0.026605866352717084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,10240,0.012106666962305706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,10240,0.02504533330599467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,1024,0.00666133314371109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,8192,0.01206719974676768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,8192,0.023931733767191567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,768,0.005980800092220307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,512,0.0056650668382644655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,7168,0.011461333433787028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,7168,0.02249386707941691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,768,128,0.004834133386611939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,6144,0.010517332951227825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,6144,0.02180479963620504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,65536,0.11878613630930583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,5120,0.010518399874369304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,51200,0.09266026814778647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,5120,0.02136533260345459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,4096,0.008624000350634257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,4096,0.019617066780726115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,4096,0.010670933127403259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,3584,0.007924266656239827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,12288,0.024918399254480996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,3584,0.010213333368301391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,3584,0.018488534291585288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,3072,0.0076682666937510175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,3072,0.01837546626726786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,2560,0.006467199822266896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,2560,0.017977599302927652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,2560,0.00865280032157898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,10240,0.020957867304484047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,2048,0.005714133381843567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,2048,0.017148800690968833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,1536,0.005050666630268097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,1536,0.016730666160583496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,1536,0.007090133428573608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,1024,0.0042367999752362575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,1024,0.016014933586120605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,768,0.003969066590070724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,8192,0.01691733400026957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,768,0.01548799971739451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,768,0.005674666663010915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,512,0.003659733384847641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,512,0.005414400001366933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,512,0.015191466609636942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,256,0.0032149332265059153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,256,0.014877866705258688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,128,0.0029877332349618276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,128,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,7168,0.015447466572125753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,64,0.0028437333802382152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,64,0.014713600277900696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,512,32,0.0028704000016053517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,512,32,0.014787200093269347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,6144,0.013914666573206582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,65536,0.02935360074043274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,65536,0.07719573179880777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,65536,0.1047327995300293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,51200,0.02424853245417277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,51200,0.06062399943669637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,16384,0.010426666339238484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,16384,0.028194133440653486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,16384,0.02796906630198161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,12288,0.010027733445167542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,12288,0.02391786575317383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,5120,0.011975466211636861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,12288,0.0223146657148997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,10240,0.01097920040289561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,10240,0.019015467166900633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,10240,0.02308373252550761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,8192,0.010194133718808491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,8192,0.022563199202219643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,7168,0.008787199854850769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,7168,0.020933334032694498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,3072,0.009258666634559631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,7168,0.014712533354759217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,6144,0.007777066528797149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,6144,0.020222934087117513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,5120,0.007266133526961009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,5120,0.020372267564137778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,5120,0.011597866813341778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,2048,0.007474133372306823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,4096,0.007259733478228251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,4096,0.01914880077044169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,4096,0.010358400146166484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,3584,0.006832000116507213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,3584,0.0184661328792572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,3584,0.010201600193977357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,3072,0.006532266736030579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,3072,0.018308266003926595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,2560,0.006221866607666016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,1024,0.006161066889762879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,2560,0.008614400029182434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,2048,0.005579733351866404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,2048,0.017059199015299478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,2048,0.007361066838105519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,1536,0.0048991998036702475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,1536,0.01629973351955414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,1536,0.007012266914049785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,1024,0.004141866664091746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,1024,0.01595946649710337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,1024,0.005971199770768484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,768,0.003786666691303253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,256,0.004940799872080485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,6144,0.013123200337092081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,768,0.00572266678015391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,768,0.01595946649710337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,512,0.0035904000202814737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,512,128,0.0047872001926104225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,256,0.00317546675602595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,256,0.014812800288200378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,2560,0.017832533518473307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,128,0.0029237332443396253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,51200,0.08309653600056967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,128,0.014803199966748556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,64,0.0028021333118279776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,64,0.014756266276041666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,256,32,0.002868266652027766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,32,0.014855466286341348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,65536,0.022054400046666464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,65536,0.07292799949645996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,8192,0.016223999857902526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,65536,0.10370666980743408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,51200,0.016661333044370015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,51200,0.05634986559549967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,256,512,0.015305599570274353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,16384,0.008493866523106892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,16384,0.025154133637746174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,12288,0.008348799745241801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,12288,0.02230613430341085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,3072,0.00900266667207082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,10240,0.007705600063006084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,10240,0.022027732928593953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,512,0.0054837331175804135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,8192,0.0070261334379514056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,8192,0.020413867632548013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,8192,0.016405333081881204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,7168,0.006664533416430156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,256,0.004993066688378652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,7168,0.020533333222071327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,6144,0.006308266520500183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,6144,0.02021226684252421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,6144,0.013123200337092081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,256,128,0.004677333434422811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,5120,0.006622933348019918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,5120,0.011667199929555257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,5120,0.02030506730079651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,4096,0.006172800064086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,3584,0.006589866677920024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,4096,0.018939733505249023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,3584,0.018433066209157307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,3072,0.006230400005976359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,3072,0.01806933283805847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,3072,0.008830933769543966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,2560,0.006098133325576782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,2560,0.01752106746037801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,2560,0.00836906631787618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,2048,0.005369600156943003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,51200,0.08157653013865153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,2048,0.017052799463272095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,2048,0.0071712002158164975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,1536,0.0046858668327331545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,1536,0.016334933042526246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,16384,0.02802133361498515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,1024,0.0040394666294256846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,1024,0.015793066223462424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,768,0.0037205333511034647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,768,0.01548693378766378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,12288,0.022321067253748574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,512,0.003455999990304311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,512,0.01544319987297058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,512,0.005299200117588043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,256,0.003014400104681651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,256,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,256,0.004916266600290934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,128,0.0028704000016053517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,128,0.014707199732462563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,10240,0.01913386583328247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,128,0.004830933113892873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,64,0.0026634665826956432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,64,0.014595199624697366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,128,32,0.0027295999228954316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,65536,0.0157258669535319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,128,32,0.014773333072662353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,51200,0.013454932967821756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,65536,0.06997866630554199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,16384,0.00726506660381953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,51200,0.05256746610005697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,12288,0.00694400022427241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,16384,0.02486506700515747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,10240,0.007006933291753133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,12288,0.02244480053583781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,8192,0.006580266853173573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,10240,0.022766933838526408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,7168,0.006306133170922597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,8192,0.020614399512608846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,7168,0.02077440023422241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,6144,0.020267732938130698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,5120,0.006421333551406861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,5120,0.01988160014152527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,4096,0.005998933315277099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,7168,0.014597333470980325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,4096,0.018918399016062418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,3584,0.0064298664530118305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,3584,0.01810986598332723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,3072,0.006126933296521505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,2560,0.005972266693909963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,3072,0.017978666226069133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,2560,0.017348267634709678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,2048,0.00543146679798762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,4096,0.01018453339735667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,2048,0.01681493322054545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,1536,0.00468800018231074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,1024,0.003982933362325033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,1536,0.016270933548609416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,768,0.003717333326737086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,1024,0.015624533096949259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,512,0.0033344000577926634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,768,0.015500799814860026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,256,0.003044266750415166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,512,0.015382400155067444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,128,0.0028170667588710784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,256,0.014822399616241455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,64,0.002678400029738744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,128,0.014575999975204468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,32,0.0026602665583292644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,64,0.014542933305104574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,65536,0.012657066186269125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,64,32,0.014637866616249084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,51200,0.010247466961542766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,65536,0.06731413205464681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,64,6144,0.006039466460545858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,16384,0.006263466676076253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,51200,0.049098666508992514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,12288,0.006949333349863689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,16384,0.0247925341129303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,1024,0.005964800218741099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,10240,0.007022933165232341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,1536,0.006988800068696339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,8192,0.006482133269309997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,10240,0.022106667359670006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,8192,0.02058346668879191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,6144,0.006029866635799408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,7168,0.020360533396402994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,6144,0.019898666938145956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,5120,0.00647680014371872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,4096,0.006080000102519989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,768,0.0056533331672350565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,5120,0.019818667570749918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,3584,0.0064746667941411335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,4096,0.01925333340962728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,2560,0.006033066908518473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,3584,0.01856213410695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,3072,0.018229333559672038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,2048,0.005341866612434387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,2560,0.01730239987373352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,1536,0.004645333190759023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,2048,0.01699413259824117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,1024,0.003988266736268997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,1536,0.016639999548594155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,1024,0.015824000040690102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,512,0.0034080001215140024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,768,0.015587199727694193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,256,0.0029909332593282064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,512,0.015292800466219583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,225,128,3584,0.009980799754460652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,128,0.002846933404604594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,256,0.014824533462524414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,128,0.014712533354759217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,64,0.0144405335187912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,32,0.014531200130780539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,225,32,12288,0.022357332706451415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,7168,0.006232533355553945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,3072,0.006067200005054474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,768,0.0036960000793139136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,64,0.0026634665826956432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,225,32,32,0.0026911998788515727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,16384,0.428761609395345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,16384,0.784504508972168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,12288,0.332967472076416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,12288,0.605624516805013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,16384,0.647596804300944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,10240,0.2811626752217611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,10240,0.5031722704569499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,12288,0.4792298634847005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,10240,0.39449707667032874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,51200,1.2890463511149088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,8192,0.23094612757364907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,51200,2.4537503560384115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,7168,0.3590559959411621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,8192,0.32696107228597004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,7168,0.20873920122782388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,7168,0.26622506777445476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,6144,0.31104853947957356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,5120,0.26173440615336097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,5120,0.15847679773966472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,6144,0.23524373372395835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,6144,0.20709652900695802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,4096,0.21408319473266602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,5120,0.19859414100646972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,4096,0.13219520250956218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,4096,0.15482452710469563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,3584,0.13361706733703613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,3584,0.13611733118693034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,3072,0.10683306852976482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,8192,0.40311253865559893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,2560,0.0926911989847819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,2560,0.15906240145365397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,2048,0.12886719703674315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,2048,0.081167999903361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,2048,0.07982079982757569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,1536,0.09142293135325114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,1536,0.0664576013882955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,1024,0.06444266637166342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,1024,0.05315200090408325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,3584,0.19049173990885418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,768,0.05807040135065714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,3072,0.16659733454386394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,768,0.04637226661046346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,768,0.04723946650822957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,512,0.03919999996821086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,512,0.04048853317896525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,512,0.03643413384755452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,256,0.027429334322611493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,256,0.032687999804814655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,256,0.031197865804036457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,128,0.019834667444229126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,128,0.028535467386245728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,128,0.028250666459401448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,64,0.014536533753077188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,65536,32,0.016023466984430947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,64,0.028547199567159016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,65536,32,0.03172373374303182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,51200,2.044026692708333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,2560,0.10058240095774333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,1024,0.04984213511149089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,65536,1.4850048065185546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,51200,1.1495957692464194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,65536,2.833698018391927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,16384,0.729596773783366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,65536,2.162792460123698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,16384,0.4044970512390137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,51200,2.2340329488118487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,12288,0.5379125595092773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,12288,0.33850132624308266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,10240,0.5028160095214844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,10240,0.2662773291269938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,8192,0.2059786637624105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,8192,0.3537376085917155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,7168,0.32659947077433266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,7168,0.18345600763956707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,1536,0.06587839921315511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,16384,0.5758111953735352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,6144,0.27042773564656575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,51200,1.6546527862548828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,12288,0.391816520690918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,65536,3072,0.1176906665166219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,6144,0.17680106163024903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,5120,0.143393071492513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,5120,0.2749504089355469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,10240,0.33563734690348307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,4096,0.11793706417083741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,4096,0.18498454093933106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,4096,0.13077226479848225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,3584,0.176363738377889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,3584,0.10776106516520183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,3072,0.0970304012298584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,8192,0.2667509396870931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,2560,0.12406506538391113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,2560,0.08871040344238282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,2048,0.1009877363840739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,2048,0.06967999935150146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,7168,0.22736746470133462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,1536,0.0794111967086792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,1536,0.059165867169698086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,3072,0.14666773478190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,1024,0.05534079869588217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,1024,0.048186667760213214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,6144,0.19830187161763507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,768,0.04858346780141194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,5120,0.1660810629526774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,768,0.042226131757100424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,768,0.0331712007522583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,512,0.03675946791966756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,3584,0.11339733600616456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,3072,0.09735146363576254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,256,0.025197867552439374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,256,0.027453867594401042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,128,0.014763733744621277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,128,0.025623466571172076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,2560,0.0817087968190511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,64,0.01316480040550232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,64,0.027559467156728107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,32,0.014897066354751586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,51200,32,0.028139734268188478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,2048,0.06378879944483438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,51200,512,0.033446399370829265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,1536,0.0487829327583313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,256,0.0225600004196167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,65536,0.45504213968912766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,1024,0.04287999868392944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,65536,0.7759712219238282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,16384,0.20107733408610023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,16384,0.14635626475016278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,51200,0.3580810546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,51200,0.6143242518107097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,51200,0.5048405329386394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,16384,0.16007359822591144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,12288,0.15601919492085775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,12288,0.1058847983678182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,10240,0.12945280075073243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,12288,0.12139093081156413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,10240,0.08846080303192139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,10240,0.10172479947408039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,8192,0.10480746428171794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,8192,0.07668906847635905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,512,0.027525333563486735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,7168,0.09294079939524333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,8192,0.08468693097432455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,6144,0.08090986410776774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,6144,0.06115520000457764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,6144,0.06295573314030965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,5120,0.0684554656346639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,5120,0.05407786766688029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,4096,0.05692693392435709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,4096,0.04678293466567993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,4096,0.04293546676635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,3584,0.05064746538798014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,3584,0.04331093231836955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,51200,128,0.01823893388112386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,3072,0.044446933269500735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,3072,0.03335893154144287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,2560,0.03802880048751831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,2560,0.03566826581954956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,2560,0.02987839976946513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,2048,0.03194453318913777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,2048,0.03128319978713989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,2048,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,1536,0.02565760016441345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,1536,0.026871466636657716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,7168,0.06742719809214273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,1024,0.020081067085266115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,1024,0.02355413238207499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,1024,0.0165503998597463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,5120,0.05407893260320028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,768,0.014173866311709086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,768,0.0216757337252299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,768,0.01528320014476776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,512,0.01051520009835561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,512,0.020451200008392335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,256,0.007865599791208903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,3584,0.03939839998881022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,256,0.018071466684341432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,256,0.010875733693440755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,128,0.00584853341182073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,3072,0.03965760072072347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,128,0.01700906753540039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,64,0.01745599905649821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,32,0.005772800246874491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,16384,32,0.018067200978597008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,65536,0.5809663772583008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,65536,0.36102186838785805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,16384,64,0.0052042668064435325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,65536,0.6458325068155925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,51200,0.27902399698893227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,51200,0.4598933219909668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,16384,0.14952425956726073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,16384,0.12177706559499105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,1536,0.021102933088938396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,12288,0.11761493682861328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,12288,0.08303893407185872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,12288,0.10520746707916259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,10240,0.09717439810434977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,10240,0.07316053708394368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,10240,0.08849706649780273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,8192,0.07978026866912842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,8192,0.06130880117416382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,128,0.009923199812571209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,7168,0.07024640242258708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,7168,0.05589226484298706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,6144,0.06184959808985392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,6144,0.05045760075251261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,65536,0.5633738835652669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,5120,0.0599669337272644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,51200,0.435533873240153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,5120,0.04649600187937419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,16384,0.139462407430013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,4096,0.043703468640645345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,4096,0.039308798313140866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,3584,0.039228800932566324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,3584,0.0370410680770874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,3584,0.03384000062942505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,3072,0.03452373345692952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,3072,0.033861335118611655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,7168,0.07282559871673584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,2560,0.02963520089785258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,16384,512,0.012916266918182373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,2560,0.0304639995098114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,2048,0.02479893366495768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,8192,0.07383146286010742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,7168,0.0639573335647583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,2048,0.026601600646972656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,1536,0.01962453325589498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,1536,0.023849600553512575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,1024,0.013571199774742127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,1024,0.02188373406728109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,6144,0.0550933321317037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,2048,0.020742400487263998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,768,0.011220266421635944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,768,0.021938133239746093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,5120,0.04771413405736287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,512,0.00956160028775533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,4096,0.038337067763010664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,512,0.019059199094772338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,256,0.006698666512966156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,256,0.017013333241144814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,3072,0.030265599489212036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,128,0.0054624001185099285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,128,0.016596266627311708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,2560,0.02486293315887451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,1536,0.01751040021578471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,64,0.004657066861788432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,12288,32,0.007495466868082683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,64,0.019197867314020792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,12288,32,0.019978666305541994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,1024,0.014367999633153281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,65536,0.34571412404378254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,65536,0.5302666664123535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,51200,0.42372798919677734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,51200,0.24685227076212563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,16384,0.14163734118143717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,16384,0.09353813330332438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,768,0.01211840013662974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,12288,0.10032107035319011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,12288,0.07503893375396728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,10240,0.08445759614308676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,10240,0.06473493178685506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,512,0.010585600137710571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,8192,0.06833279927571614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,256,0.008492799599965413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,8192,0.055118934313456214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,7168,0.060519464810689295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,12288,128,0.00780266672372818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,7168,0.0508565346399943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,6144,0.053428268432617186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,6144,0.046801066398620604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,65536,0.5060970624287923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,5120,0.05215040047963461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,5120,0.04285866816838582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,51200,0.39969921112060547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,4096,0.041918933391571045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,4096,0.03709226846694946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,16384,0.1297600030899048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,12288,0.09659626483917236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,3584,0.03425706624984741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,3584,0.03448853492736816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,3072,0.03012053370475769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,3072,0.031828266382217404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,10240,0.08080213069915772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,2560,0.025983999172846477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,2560,0.028253867228825884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,8192,0.06874240239461263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,2048,0.022311466932296752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,2048,0.02578986684481303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,7168,0.05899200042088827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,1536,0.017866667111714682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,1536,0.023884799083073935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,1536,0.01574613352616628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,6144,0.050756267706553136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,1024,0.01334933340549469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,1024,0.021035732825597127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,768,0.010311466455459595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,768,0.019935999313990274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,5120,0.04388373295466105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,512,0.008680533369382222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,512,0.01764799952507019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,512,0.00921066701412201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,4096,0.035011200110117595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,256,0.016989866892496742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,3584,0.03102506597836812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,128,0.0052159999807675685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,128,0.016200533509254454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,3072,0.02768213351567586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,64,0.004530133306980133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,64,0.01621119976043701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,32,0.004548266530036926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,2560,0.022712532679239908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,10240,32,0.016503467162450155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,65536,0.2750186602274577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,65536,0.4096192042032878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,2048,0.01997013290723165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,10240,256,0.006088533500830332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,1024,0.013143466909726462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,51200,0.32583786646525065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,51200,0.2182975927988688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,16384,0.10796799659729003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,768,0.011241599917411804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,51200,0.26177387237548827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,12288,0.08422933419545492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,12288,0.06575786670049032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,16384,0.09132906595865885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,10240,0.07417279879252116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,8192,0.05708160003026327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,10240,0.05788480043411255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,10240,0.053732268015543615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,8192,0.046537601947784425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,7168,0.05215573310852051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,7168,0.04307626485824585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,256,0.007698133091131846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,6144,0.04606613318125407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,6144,0.03951679865519206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,10240,128,0.007092266778151194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,6144,0.034246400992075605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,5120,0.03572266499201457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,5120,0.029263999064763385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,4096,0.02960639993349711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,4096,0.031172267595926922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,65536,0.331114673614502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,3584,0.029986133178075153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,16384,0.08531626860300699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,5120,0.039858134587605794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,3584,0.028696533044179278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,12288,0.0624725341796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,3072,0.023817600806554158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,3072,0.026332799593607587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,2560,0.020499199628829956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,2560,0.024382932980855306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,8192,0.04461013476053874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,2048,0.016922666629155477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,2048,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,2048,0.02301759918530782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,1536,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,1536,0.02140799959500631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,1536,0.013438933094342551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,7168,0.038873600959777835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,1024,0.009964799880981446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,1024,0.019387733936309815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,768,0.008520533641179402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,512,0.0070720002055168155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,768,0.01813653310139974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,512,0.017214934031168617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,512,0.008474666873613994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,256,0.004819199939568837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,256,0.016261333227157594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,128,0.004293333490689596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,128,0.015802666544914246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,128,0.007130666573842366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,64,0.00396373321612676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,64,0.01620693306128184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,8192,32,0.0042133331298828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,8192,32,0.016687999169031777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,65536,0.3728543917338053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,4096,0.024501333634058632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,65536,0.2566666603088379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,65536,0.31216532389322915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,51200,0.29487040837605794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,51200,0.21989866892496743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,16384,0.11409173011779786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,3584,0.023202133178710938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,16384,0.07661973635355632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,3072,0.020459733406702676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,12288,0.08411946296691894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,12288,0.0616096019744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,10240,0.06960533459981283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,10240,0.0534880002339681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,1024,0.011054933071136475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,768,0.01009173293908437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,8192,0.05435520013173421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,256,0.007335466643174489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,8192,0.04239893356959025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,8192,0.04399253527323405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,7168,0.04709759950637817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,7168,0.03752426703770955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,7168,0.04049386580785115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,6144,0.0424511988957723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,5120,0.03489706516265869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,5120,0.0335370659828186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,5120,0.028546132644017536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,4096,0.02684906721115112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,4096,0.029262934128443403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,8192,2560,0.017489065726598106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,3584,0.024486400683720908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,3584,0.02781013250350952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,6144,0.037581864992777506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,3072,0.021731199820836385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,3072,0.02545386751492818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,51200,0.24502506256103515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,2560,0.020940800507863365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,16384,0.08152320384979247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,2560,0.02431679964065552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,2560,0.016745599110921223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,12288,0.059463465213775636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,2048,0.02214933236440023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,1536,0.011979732910792034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,1536,0.020812799533208214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,10240,0.05140053431193033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,1024,0.00937493344148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,1024,0.01904746691385905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,6144,0.032289065917332965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,768,0.008060800035794576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,768,0.016922666629155477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,4096,0.023963733514149984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,512,0.006604800124963124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,512,0.017138133446375527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,512,0.007380266487598419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,256,0.004738133152325948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,256,0.016311466693878174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,2048,0.01490239997704824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,3584,0.02069546580314636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,128,0.004163199911514918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,128,0.015714133779207863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,3072,0.01919040083885193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,64,0.003706666578849157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,64,0.01568000018596649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,7168,32,0.003774933268626531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,7168,32,0.015685333808263143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,65536,0.3316245396931966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,2048,0.01434879998366038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,1536,0.012126933534940083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,65536,0.22367040316263834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,65536,0.28652801513671877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,1024,0.01014400025208791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,51200,0.2674250602722168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,51200,0.19783573150634765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,16384,0.10279040336608887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,768,0.008521599570910136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,12288,0.07625919977823893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,16384,0.07228373686472575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,12288,0.05837440093358358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,12288,0.057127467791239415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,10240,0.05023893515268961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,10240,0.04850560029347738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,8192,0.05018666585286459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,8192,0.046212267875671384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,7168,0.04649920066197713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,7168,0.0406335989634196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,256,0.006546133259932201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,6144,0.03938133319218953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,6144,0.030371199051539104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,6144,0.03704426685969035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,5120,0.03303253253300985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,5120,0.03361920118331909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,5120,0.026954666773478193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,4096,0.027913600206375122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,4096,0.029813333352406816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,3584,0.024678399165471397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,3584,0.027317333221435546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,3584,0.020197333892186482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,3072,0.021466666460037233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,3072,0.0254805326461792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,10240,0.06153066555658976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,3072,0.018237866957982383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,2560,0.018755199511845906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,2560,0.02416960000991821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,2048,0.015382400155067444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,2560,0.01633386711279551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,2048,0.01367573340733846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,1536,0.012251733740170797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,1536,0.021005866924921672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,7168,128,0.006039466460545858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,1024,0.009324799974759419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,1024,0.017781333128611247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,2048,0.022308266162872313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,51200,0.2256927967071533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,16384,0.0747648000717163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,768,0.007805866499741872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,512,0.005965866645177205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,768,0.008295466502507527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,768,0.017760000626246133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,512,0.00728000005086263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,256,0.0047199999292691554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,256,0.006499200065930684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,256,0.016004266341527303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,128,0.004144000013669332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,8192,0.03960640033086141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,128,0.015770666797955833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,64,0.0036917333801587426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,6144,32,0.0037493333220481873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,64,0.015569067001342774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,32,0.015698132912317912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,7168,0.03494079907735188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,65536,0.2715093294779459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,65536,0.20533547401428223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,4096,0.02255893349647522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,65536,0.2672085444132487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,51200,0.21408534049987793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,16384,0.06799039840698243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,16384,0.06546346743901571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,12288,0.05370986859003703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,12288,0.05084906816482544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,6144,512,0.016706132888793947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,12288,0.05153599977493286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,10240,0.05177173217137655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,10240,0.04482453266779582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,8192,0.04094933271408081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,1536,0.011635200182596842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,8192,0.03980266650517782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,7168,0.037298134962717694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,7168,0.03646080096562703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,51200,0.1623978614807129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,1024,0.009220266342163086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,6144,0.02913813392321269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,6144,0.02807253400484721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,6144,0.03341653347015381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,5120,0.027639466524124145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,5120,0.029917865991592407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,5120,0.02368639906247457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,4096,0.022402133544286093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,4096,0.026954666773478193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,6144,128,0.006064000229040781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,3584,0.018820265928904213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,3584,0.026040534178415935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,3584,0.01884160041809082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,3072,0.01801066597302755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,3072,0.02448106606801351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,3072,0.016404267152150473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,2560,0.013970133662223817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,2560,0.022516266504923502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,51200,0.20626880327860514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,2048,0.013477333386739097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,16384,0.07004160086313883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,2048,0.021397332350413002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,1536,0.010897066195805867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,1536,0.019990400473276774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,10240,0.04497919877370198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,1024,0.007653333246707916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,1024,0.018132267395655315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,8192,0.037164799372355145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,768,0.0072405333320299785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,768,0.017080533504486083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,7168,0.03256746729214986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,512,0.0058378666639328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,512,0.017072000106175742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,4096,0.020779732863108316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,256,0.00456639975309372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,256,0.016156799594561257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,128,0.004056533426046371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,128,0.015871999661127727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,128,0.005577600002288819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,64,0.003583999971548716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,64,0.015539200107256571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,5120,32,0.0037205333511034647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,5120,32,0.015575466553370157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,2560,0.015051733454068503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,65536,0.2184277375539144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,65536,0.17686079343159994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,2048,0.012487467130025227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,51200,0.1733173370361328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,1536,0.01057919959227244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,16384,0.06505706707636515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,51200,0.1411285400390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,1024,0.008980266253153483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,16384,0.06438613335291545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,12288,0.05260159969329834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,12288,0.05252373218536377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,10240,0.05007466475168863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,768,0.007691733539104462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,10240,0.04082346757253011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,8192,0.03994880119959514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,512,0.006818133095900218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,5120,256,0.005894400179386139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,7168,0.03543999989827474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,7168,0.03141226569811503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,8192,0.03498133420944214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,6144,0.03150399923324585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,6144,0.028909866015116376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,65536,0.24290666580200196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,5120,0.023075199127197264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,5120,0.026655999819437663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,51200,0.18661333719889323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,5120,0.022374399503072104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,4096,0.017826133966445924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,4096,0.024796799818674723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,16384,0.061493333180745444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,3584,0.01602666676044464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,12288,0.04791146516799927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,3584,0.02332373261451721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,3072,0.014542933305104574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,3072,0.022258132696151733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,10240,0.04041279951731364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,2560,0.012770133217175803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,2560,0.013929599523544311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,2560,0.021176532904307047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,2048,0.010801066954930622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,2048,0.02012053330739339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,2048,0.012050132950146992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,1536,0.008759466807047527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,1536,0.018769067525863648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,1024,0.007132799923419952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,8192,0.03359359900156657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,1024,0.016821332772572837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,768,0.006146133442719777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,768,0.016557866334915163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,7168,0.029449599981307983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,512,0.004766933123270671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,512,0.016552533706029257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,512,0.00680320014556249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,256,0.004053333401679992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,256,0.015554133057594299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,6144,0.025922133525212603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,128,0.0036373332142829893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,128,0.015214932958285013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,4096,0.018935465812683107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,64,0.0033813332517941795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,64,0.015745066603024802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,3584,0.01763733426729838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,4096,32,0.003619199991226196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,3072,0.016178133090337117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,4096,32,0.01574720044930776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,65536,0.1958954652150472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,65536,0.16582293510437013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,1536,0.010494933525721232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,1024,0.008338133494059246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,51200,0.15720319747924805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,51200,0.13197440306345623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,768,0.0076447998483975725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,16384,0.056302932898203525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,16384,0.05360639890034994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,51200,0.18027092615763346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,12288,0.045688533782958986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,12288,0.04454826513926188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,10240,0.036373333136240645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,12288,0.045254401365915936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,10240,0.03947306474049886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,8192,0.02972480058670044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,10240,0.03967146476109822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,8192,0.03187413414319356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,7168,0.02690666715304057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,7168,0.02914239962895711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,6144,0.024001065889994302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,6144,0.02752959926923116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,6144,0.02542080084482829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,5120,0.02104960083961487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,5120,0.02550506591796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,256,0.00625493327776591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,4096,0.01556373337904612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,4096,0.02287893295288086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,4096,128,0.006098133325576782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,3584,0.014095999797185264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,3584,0.022267733017603555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,3584,0.016450132926305136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,3072,0.012235732873280843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,3072,0.0214901328086853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,3072,0.015201066931088766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,2560,0.010800000031789143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,2560,0.020322134097417198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,2048,0.00928106705347697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,2048,0.019485867023468016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,65536,0.22869226137797036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,1536,0.007965866724650066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,1536,0.01839146614074707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,16384,0.060897068182627356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,1024,0.005894400179386139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,1024,0.0162581334511439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,8192,0.03269226749738057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,1024,0.007979733248551685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,7168,0.02876480023066203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,768,0.005064533154169718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,512,0.004435199995835622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,768,0.016284799575805663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,5120,0.02109866738319397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,256,0.0037962667644023894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,256,0.015631999572118124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,4096,0.018626133600870766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,128,0.003500800083080927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,128,0.015110400319099427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,512,0.016306133071581522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,2560,0.013174399733543396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,64,0.0031797334551811217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3584,32,0.003270400067170461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,2048,0.011326932907104492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,64,0.01504853367805481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3584,32,0.015262933572133383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,1536,0.00981333355108897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,65536,0.17053759892781575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,51200,0.13710187276204427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,65536,0.1548085371653239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,16384,0.05070933500925699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,51200,0.12264426549275716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,768,0.007208533088366191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,16384,0.05692160129547119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,512,0.006364800035953522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,12288,0.04949653148651123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,12288,0.04621866544087728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,10240,0.043239466349283856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,12288,0.0438645323117574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,10240,0.040302932262420654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,256,0.005709866682688395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,10240,0.03811306556065877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,8192,0.034825599193573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,7168,0.03094506661097209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,7168,0.02877333362897237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,6144,0.025754666328430174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,6144,0.026807467142740887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3584,128,0.00543146679798762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,5120,0.021971199909845987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,5120,0.024477867285410564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,8192,0.03464213212331136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,4096,0.018311466773351034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,4096,0.022882133722305298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,65536,0.21462079683939614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,3584,0.013026133179664612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,3584,0.021997867027918498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,16384,0.056279468536376956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,3072,0.011601066589355469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,3072,0.021248000860214233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,8192,0.03160640001296997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,7168,0.027266132831573486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,2560,0.010219732920328777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,2560,0.02037546634674072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,6144,0.02363306681315104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,2048,0.008846933643023174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,2048,0.01880319913228353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,5120,0.019887999693552653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,1536,0.00763626645008723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,1536,0.017310933272043864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,1536,0.008901333808898926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,1024,0.005614933371543884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,1024,0.016679465770721436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,51200,0.17088319460550944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,768,0.004923733572165171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,768,0.01599253316720327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,768,0.006741333504517872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,512,0.00436160018046697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,512,0.01607360045115153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,4096,0.017682133118311565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,256,0.0038346665600935614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,256,0.015658666690190635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,3072,0.014231466253598533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,128,0.003505066782236099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,3584,0.016210132837295534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,128,0.01518933375676473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,64,0.003269333392381668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,64,0.015009066462516785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,3072,32,0.003345066557327906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,3072,32,0.015176533659299215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,65536,0.1484106699625651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,65536,0.14066133499145508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,51200,0.11912319660186768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,2560,0.01237546702226003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,2048,0.010624000430107116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,51200,0.11157653331756592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,16384,0.044684799512227376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,16384,0.04628373384475708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,1024,0.0074869334697723385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,512,0.00603413333495458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,12288,0.035860268274943034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,10240,0.031362134218215945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,12288,0.038279465834299725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,10240,0.038734932740529374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,8192,0.023085866371790567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,256,0.0055637334783871974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,8192,0.033411200841267905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,7168,0.020810665686925252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,7168,0.03083840012550354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,51200,0.16495679219563802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,6144,0.01792959968249003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,6144,0.028564266363779706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,3072,128,0.005118933320045471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,5120,0.020777599016825358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,5120,0.024216532707214355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,5120,0.019688532749811808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,4096,0.01775146722793579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,4096,0.022590933243433635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,65536,0.20446720123291015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,3584,0.015939199924468996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,3584,0.0215338667233785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,16384,0.05526400009791056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,3072,0.0141184002161026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,3072,0.02034453352292379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,12288,0.0424074649810791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,2560,0.010233599940935771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,2560,0.019810134172439577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,8192,0.02889066735903422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,2048,0.009116799632708231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,2048,0.01790293256441752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,7168,0.025689599911371867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,1536,0.007339733342329661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,1536,0.017566933234532674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,6144,0.022320000330607097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,1024,0.005628799895445505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,1024,0.016321067015329996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,10240,0.035234133402506515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,768,0.005027199784914652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,768,0.016193067034085594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,4096,0.016910932461420693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,512,0.004330666859944662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,512,0.0159850666920344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,3584,0.015638400117556253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,256,0.0037439999481042228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,256,0.015271466970443726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,3072,0.013858133554458618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,2560,0.012365866700808208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,128,0.003352533280849457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,2048,0.010066133737564088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,128,0.015430399775505066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,1536,0.008753066261609394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,64,0.003222399950027466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2560,32,0.003205333401759466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,64,0.015155200163523355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,1024,0.007473066449165344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2560,32,0.014945066968599954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,65536,0.12056213219960529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,65536,0.1255680004755656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,768,0.006806399921576183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,51200,0.09668479760487875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,16384,0.03816106716791789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,16384,0.05121493339538574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,512,0.0060149331887563075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,12288,0.0420415997505188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,12288,0.04095466534296672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,256,0.005558399856090546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,51200,0.10004159609476726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,10240,0.03459200064341227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,10240,0.035785599549611406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,8192,0.02677546739578247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,8192,0.031787733236948654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2560,128,0.005226666728655497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,51200,0.14769280751546224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,7168,0.02908266584078471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,65536,0.19865280787150066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,6144,0.020537600914637247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,6144,0.026841600735982258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,6144,0.021571199099222817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,5120,0.017661867539087932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,16384,0.05009706815083822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,5120,0.023475199937820435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,4096,0.014801067113876343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,4096,0.021887999773025513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,3584,0.012552533547083536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,12288,0.03877653280893962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,3584,0.021134932835896812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,7168,0.025639466444651288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,3072,0.011370666821797689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,3072,0.01962239940961202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,2560,0.009246933460235595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,2560,0.018289067347844443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,10240,0.0333024005095164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,2048,0.008001066744327545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,2048,0.01745706597963969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,2048,0.009815466403961182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,1536,0.006311466793219249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,1536,0.016884267330169678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,8192,0.028408533334732054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,1024,0.004853333532810211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,1024,0.016178133090337117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,7168,0.024660267432530723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,768,0.004321066538492838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,768,0.01601066688696543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,5120,0.019028266270955406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,512,0.003832533210515976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,512,0.015577600399653117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,512,0.005854933460553487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,256,0.003453866640726725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,256,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,4096,0.01585813363393148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,128,0.0032586666444937387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,128,0.014778666694959006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,3584,0.015319466590881348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,128,0.0052042668064435325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,64,0.0030613332986831666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,2048,32,0.0033141332368055976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,32,0.01525973379611969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,65536,0.09776960213979086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,2560,0.011557333668073018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,65536,0.1105024019877116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,51200,0.0729429324467977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,51200,0.08687787055969239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,1536,0.008573866883913676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,16384,0.029753599564234418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,16384,0.03950506846110026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,2048,64,0.014963199694951376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,12288,0.028257066011428834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,1024,0.007159466544787089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,3072,0.01295253336429596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,10240,0.024889600276947022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,12288,0.03219626744588216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,10240,0.02950506607691447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,8192,0.017869865894317626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,8192,0.026601600646972656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,768,0.006614399949709575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,7168,0.015773866573969522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,7168,0.024819199244181314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,2048,256,0.005513600011666616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,6144,0.014100266496340432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,6144,0.023803732792536416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,65536,0.18321706453959147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,5120,0.016515200336774193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,5120,0.021705599625905354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,5120,0.017313067118326822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,4096,0.013977600137392679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,4096,0.02030506730079651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,16384,0.04626453320185343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,3584,0.011766399939854939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,3584,0.01939093271891276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,12288,0.036686933040618895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,3072,0.010589866836865743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,10240,0.031274666388829545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,3072,0.01850559910138448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,2560,0.007657599945863088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,2560,0.01811093290646871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,8192,0.025320533911387128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,2048,0.006637866795063019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,2048,0.01747093399365743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,7168,0.022872533400853476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,51200,0.1468213399251302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,1536,0.005610666672388713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,1536,0.016808533668518068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,6144,0.019556266069412232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,1024,0.004713599880536398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,1024,0.01630506714185079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,4096,0.015225600202878317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,768,0.004308266441027323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,768,0.015491200486818948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,768,0.006442666550477346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,512,0.0038463999827702843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,3584,0.013564800222714743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,512,0.015660799543062844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,256,0.0034634667138258614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,256,0.015044266978899637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,3072,0.012183466553688049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,128,0.0031850665807724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,2560,0.01074133316675822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,128,0.015026133259137472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,64,0.0030389333764712016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,64,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1536,32,0.0030080000559488933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,2048,0.009147733449935913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1536,32,0.01495146652062734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,65536,0.06918293635050456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,65536,0.09739627043406168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,51200,0.05624746481577555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,1536,0.008039466540018718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,16384,0.02553279995918274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,51200,0.07681492964426676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,16384,0.03697280089060466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,12288,0.02346773346265157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,12288,0.03160213430722554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,1024,0.007004799942175548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,10240,0.020310399929682414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,10240,0.02861119906107585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,512,0.005834666887919108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,8192,0.016099199652671814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,256,0.0052928000688552855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,8192,0.025465599695841473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,7168,0.014220800002415976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,7168,0.024244266748428344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1536,128,0.005060266455014547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,6144,0.013493333260218301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,6144,0.023125332593917847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,65536,0.1650965372721354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,5120,0.013309866189956665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,5120,0.02243306636810303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,5120,0.016743467251459757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,51200,0.13021866480509442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,4096,0.01086293359597524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,16384,0.04496959845225017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,4096,0.020398932695388793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,3584,0.010206933816274006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,3584,0.01971093416213989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,3072,0.008065066734949748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,3072,0.019377066691716512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,3072,0.011802666385968526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,2560,0.008725333213806152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,2560,0.018284799655278523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,10240,0.029406932989756267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,2048,0.007546666761239369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,2048,0.01769919991493225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,12288,0.034833065668741864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,1536,0.005830400188763936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,1536,0.016934400796890257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,8192,0.02534826596577962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,1024,0.005129600067933401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,1024,0.01579093337059021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,7168,0.022106667359670006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,768,0.0044266665975252785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,768,0.015637333194414772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,6144,0.01972800095876058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,512,0.0039264000952243805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,512,0.015500799814860026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,4096,0.014120533068974813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,256,0.0034773332377274835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,3584,0.013538133104642233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,256,0.014929067095120749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,128,0.0031680000325044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,128,0.014837333559989929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,128,0.004907733201980591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,64,0.002993066608905792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,64,0.014826666315396628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,1024,32,0.003045333425203959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,2560,0.010541866223017376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,1024,32,0.014906666676203408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,65536,0.0553706685702006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,2048,0.009034666419029235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,65536,0.09107946554819743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,51200,0.04575146834055583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,51200,0.07124799887339274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,16384,0.019525333245595296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,51200,0.10649173259735108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,16384,0.031905066967010495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,12288,0.015998933712641397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,12288,0.0298250675201416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,1536,0.00790826678276062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,10240,0.016387200355529784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,10240,0.026964267094930012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,1024,0.006903466582298279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,8192,0.013236266374588013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,768,0.006281599899133046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,8192,0.024615466594696045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,512,0.005767466624577841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,7168,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,7168,0.024094933271408082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,1024,256,0.005245866874853769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,6144,0.012258133292198181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,6144,0.021964800357818604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,6144,0.015373866756757101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,5120,0.010826667149861652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,5120,0.02188373406728109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,65536,0.13186240196228027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,4096,0.009378133217493693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,4096,0.02051946719487508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,16384,0.03500373363494873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,3584,0.008529067039489746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,3584,0.01983573238054911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,3584,0.011546666423479717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,12288,0.027727999289830524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,3072,0.007566933333873749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,3072,0.019189333915710448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,2560,0.006629333396752675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,2560,0.018001067638397216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,10240,0.02328959902127584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,2048,0.005725866556167603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,2048,0.01713599960009257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,2048,0.007946666578451793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,1536,0.005127466718355815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,1536,0.016673066218694053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,8192,0.020374399423599244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,1024,0.004458666841189066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,1024,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,768,0.004126933217048645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,768,0.01565120021502177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,7168,0.017620267470677693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,512,0.003702399879693985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,512,0.015560533603032431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,512,0.005414400001366933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,256,0.0033312000334262846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,256,0.015171200037002563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,5120,0.013645866513252258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,128,0.0030048000315825146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,128,0.01506239970525106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,4096,0.011633066336313884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,64,0.002922666569550832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,64,0.014737066626548768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,768,32,0.002935466667016347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,768,32,0.014842666188875833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,3072,0.010041600465774536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,65536,0.04262719949086507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,65536,0.08603413105010986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,65536,0.11530880133310954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,51200,0.03525333404541016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,2560,0.009337600072224934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,16384,0.016261333227157594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,51200,0.06971413294474284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,16384,0.029286400477091475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,12288,0.013035733501116434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,12288,0.025893332560857137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,1536,0.007386666536331177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,10240,0.012049067020416259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,10240,0.023800534009933472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,1024,0.0063360000650088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,10240,0.020797866582870483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,8192,0.010583466291427613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,768,0.0059562668204307554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,8192,0.022275199492772423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,8192,0.01677653392155965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,7168,0.011639466881752015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,7168,0.014844800035158793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,7168,0.022523732980092366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,6144,0.00902186632156372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,5120,0.008431999882062276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,6144,0.013664000233014426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,6144,0.021346133947372437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,4096,0.008389332890510559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,5120,0.020631466309229532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,256,0.0051584000388781226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,4096,0.019012266397476198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,3584,0.007960533102353413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,3584,0.01834986607233683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,3072,0.007341866691907247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,3072,0.01812373399734497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,768,128,0.004817066589991251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,2560,0.006386133531729381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,2560,0.017694934209187826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,51200,0.08987200260162354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,2048,0.005691733459631602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,16384,0.03107306758562724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,2048,0.017119999726613364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,2048,0.007604266703128815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,1536,0.005038933455944061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,1536,0.006941866874694824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,3072,0.009195733070373534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,1024,0.004368000229199727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,12288,0.024439465999603272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,1024,0.015922133127848306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,768,0.00393599991997083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,768,0.015381333231925965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,512,0.0036458666125933326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,5120,0.011850666999816895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,4096,0.010609066486358643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,1536,0.016668800512949625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,512,0.01511146624883016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,256,0.0032960000137488045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,256,0.0050122668345769245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,128,0.002946133414904277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,128,0.01511360009511312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,2560,0.008631466825803121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,64,0.0028351999819278715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,64,0.01453013320763906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,512,32,0.0028213332096735638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,32,0.014574933052062988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,65536,0.03203306595484416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,65536,0.07663040161132813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,512,256,0.014942933122316995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,3584,0.010270933310190838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,51200,0.02610986630121867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,51200,0.060619731744130455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,1024,0.006206933160622915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,16384,0.010845866799354554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,16384,0.028488532702128096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,16384,0.02617066701253255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,12288,0.010941867033640544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,768,0.005820799867312113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,12288,0.02332373261451721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,12288,0.022100265820821127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,10240,0.009798399607340495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,8192,0.00865280032157898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,10240,0.022273067633310953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,8192,0.02104746699333191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,7168,0.0077450667818387345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,7168,0.020987733205159505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,7168,0.014603733023007711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,6144,0.007249066730340321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,6144,0.019552000363667808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,5120,0.006805333495140076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,5120,0.020992000897725425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,512,0.0052714665730794275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,4096,0.006393600006898243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,4096,0.01907520095507304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,4096,0.010437333583831787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,3584,0.006739200154940288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,3584,0.01880426605542501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,512,128,0.004790399968624115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,3072,0.006387199958165486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,3072,0.0181877334912618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,3072,0.009052800138791402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,2560,0.006293333570162455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,2560,0.017505067586898803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,2560,0.008659199873606364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,2048,0.005599999924500784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,2048,0.016960000991821288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,2048,0.007375999788443248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,1536,0.004891733328501383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,1536,0.01653333306312561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,51200,0.08059733708699544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,1024,0.004145066688458125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,3584,0.010051199793815612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,1024,0.015853866934776306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,768,0.0037823999921480812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,768,0.01558080017566681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,8192,0.01641279955705007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,10240,0.019299199183781944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,512,0.003562666724125544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,512,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,512,0.00528106689453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,256,0.003173333406448364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,256,0.005010133484999338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,128,0.0029311999678611755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,6144,0.013205333550771078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,128,0.014703999956448874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,128,0.004689066608746847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,65536,0.10323946475982666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,64,0.0028223998844623564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,256,32,0.0028170667588710784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,64,0.014749866724014283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,32,0.014761599898338317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,65536,0.02158400019009908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,51200,0.014696533481280008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,5120,0.011509333054224651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,51200,0.05371306737263998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,51200,0.08198186556498209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,16384,0.008473599950472515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,16384,0.024820266167322795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,12288,0.00784853349129359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,12288,0.021901865800221763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,1024,0.0061034664511680605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,10240,0.007170133292675018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,10240,0.022086399793624877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,256,256,0.015013333161671957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,8192,0.006706133484840393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,8192,0.02053546706835429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,65536,0.07160320281982421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,7168,0.006410666803518932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,7168,0.0209279994169871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,7168,0.014689067006111145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,6144,0.006180266539255777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,6144,0.020124799013137816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,1536,0.0070250665148099255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,256,768,0.00561599979797999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,5120,0.006589866677920024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,5120,0.02021119991938273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,5120,0.011478400230407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,4096,0.006218666831652323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,4096,0.01890666683514913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,65536,0.1047434647878011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,3584,0.006687999765078227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,16384,0.02801919976870219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,3584,0.018478933970133463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,3072,0.006379733482996623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,3072,0.018398932615915933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,12288,0.021938133239746093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,2560,0.006089599927266439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,2560,0.017318399747212727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,2560,0.00827946662902832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,2048,0.005390933156013489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,2048,0.016673066218694053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,10240,0.019045333067576088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,1536,0.004781866570313772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,1536,0.016328533490498863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,8192,0.016403200229008992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,1024,0.004137599964936575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,1024,0.015777066349983215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,1024,0.00619946668545405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,768,0.0037258667250474296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,768,0.015384533007939658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,768,0.005681066711743673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,512,0.00346666673819224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,512,0.015180800358454385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,512,0.005330133438110352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,256,0.0030378667016824085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,6144,0.013114666938781739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,256,0.004972800115744273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,256,0.014756266276041666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,128,0.002902399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,128,0.014591999848683677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,64,0.0027562665442625684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,64,0.014411733547846476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,128,32,0.0027637332677841187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,128,32,0.014591999848683677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,65536,0.01390506625175476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,51200,0.011358933647473653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,65536,0.06864853700002035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,16384,0.00709440012772878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,51200,0.05114986499150594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,16384,0.024682666858037314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,12288,0.006378666559855144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,10240,0.006623999774456024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,12288,0.0224671999613444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,8192,0.0064650664726893115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,10240,0.022077866395314536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,7168,0.00624533345301946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,8192,0.02062826752662659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,6144,0.00606826643149058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,7168,0.020558933417002358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,5120,0.006550399959087372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,6144,0.019801600774129232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,4096,0.006075733403364817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,5120,0.01995840072631836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,4096,0.010190932949384054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,3584,0.006545066833496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,4096,0.018745599190394084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,3072,0.006222933530807495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,3584,0.018583466609319053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,2560,0.006095999975999197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,3584,0.00985706647237142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,2048,0.00543146679798762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,2560,0.017552000284194947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,1536,0.0047189335028330484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,2048,0.016921599706014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,1024,0.003991466760635376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,1536,0.016386133432388306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,768,0.0036330667634805044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,3072,0.009060266613960265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,768,0.015795200069745382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,256,0.0029685333371162414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,512,0.015286399920781454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,128,0.0028138667345046996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,256,0.014868266383806863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,64,0.002647466709216436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,128,0.015233066678047181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,32,0.002682666728893916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,64,0.014667733510335287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,65536,0.012536533673604331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,32,0.014717866977055868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,51200,0.010088533163070679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,65536,0.06694186528523763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,2048,0.007302399973074596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,51200,0.049540265401204424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,16384,0.00625600020090739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,12288,0.006226133306821187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,16384,0.024654932816823325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,1536,0.006908800204594929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,12288,0.022270933787027995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,8192,0.006291200220584869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,10240,0.021713066101074218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,7168,0.006186666587988535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,8192,0.021285333236058555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,6144,0.00598826656738917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,7168,0.021253333489100138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,6144,0.019747199614842732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,5120,0.006279466549555461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,4096,0.005904000004132589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,5120,0.020137600104014077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,3584,0.006313600142796834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,3584,0.018126932779947917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,3072,0.006027733286221823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,3072,0.018130133549372353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,2560,0.0060576001803080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,3072,0.017796266078948974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,2048,0.005388799806435903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,2560,0.017516799767812095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,64,1024,0.015756799777348836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,1536,0.004692266881465912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,2048,0.017051732540130614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,64,512,0.0034506666163603462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,1024,0.004078933348258337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,768,0.0037471999724706015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,1024,0.015415466825167336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,512,0.003385599950949351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,768,0.015332266688346863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,256,0.0029994666576385496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,512,0.015041066209475198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,128,0.0028053333361943563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,256,0.014632532993952433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,224,128,128,0.004757333298524221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,64,0.0026848000784715016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,128,0.014633599917093912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,32,0.0026410666604836782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,32,0.014674133062362671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,224,32,10240,0.006354133288065593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,4096,0.018967467546463012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,1536,0.016385066509246825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,16384,0.7749151865641276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,224,32,64,0.014447999993960061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,16384,0.41576000849405925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,12288,0.5920448303222656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,51200,1.255395253499349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,16384,0.7032981236775716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,12288,0.32642666498819983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,12288,0.5078250567118328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,10240,0.4976042747497559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,8192,0.22431573867797852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,10240,0.27431891759236654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,8192,0.3950101216634115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,51200,2.4238377888997396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,8192,0.36096213658650717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,7168,0.19938453038533527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,6144,0.305513604482015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,7168,0.34910081227620443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,7168,0.3227381388346354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,6144,0.2719264030456543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,5120,0.25578880310058594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,5120,0.1519829273223877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,4096,0.1279306650161743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,4096,0.20722667376200357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,5120,0.2294314702351888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,6144,0.19681599934895833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,3584,0.11517972946166992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,3584,0.1835466702779134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,3072,0.16002666155497233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,4096,0.1792629400889079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,3584,0.15655040740966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,3072,0.11138880252838135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,2560,0.13618133862813314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,2048,0.11170986493428547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,2560,0.08892587025960287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,2560,0.11410773595174153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,1536,0.08668159643809001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,2048,0.0756053368250529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,2048,0.09151786963144938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,1536,0.06351573467254638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,1536,0.06725333531697592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,1024,0.06209493478139242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,768,0.05014933347702026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,1024,0.04965759913126628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,768,0.04504426717758179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,512,0.03779093424479167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,256,0.02495680054028829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,512,0.03790719906489055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,256,0.030679466327031453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,128,0.01704533298810323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,128,0.026578134298324584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,1024,0.055043200651804604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,64,0.013428266843159994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,64,0.02728853424390157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,65536,32,0.014971733093261719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,51200,2.2342432657877604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,65536,32,0.030403200785319013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,3072,0.13514879544576008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,512,0.03573226531346639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,128,0.027705599864323933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,65536,1.4189418792724608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,768,0.04621333281199137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,65536,2.7866411844889325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,10240,0.46171948115030925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,51200,1.1351103464762369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,65536,256,0.03027519981066386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,16384,0.6845087687174479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,51200,2.220750935872396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,12288,0.28865814208984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,16384,0.4227402687072754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,12288,0.5308671951293945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,16384,0.5737621307373046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,10240,0.43523305257161454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,8192,0.34768959681193035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,10240,0.2746197382609049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,8192,0.20053760210673013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,7168,0.3111978530883789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,7168,0.17928959528605143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,7168,0.27637974421183265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,6144,0.26819839477539065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,6144,0.15618453025817872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,5120,0.22569066683451333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,5120,0.13087999820709229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,4096,0.1923807938893636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,65536,2.267897542317708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,4096,0.1095018704732259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,3584,0.16085227330525714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,3584,0.0989525318145752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,12288,0.4568789482116699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,10240,0.3677706718444824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,3072,0.14841492970784503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,3072,0.08776533603668213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,2560,0.11675413449605306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,8192,0.30782079696655273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,2560,0.07687679926554361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,2048,0.09475839932759603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,2048,0.06678933302561442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,51200,1.8624565124511718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,6144,0.22998080253601075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,1536,0.08004799683888754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,1536,0.05517119963963827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,1024,0.05145493348439535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,1024,0.04510613282521565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,5120,0.1925098737080892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,768,0.04216426610946655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,768,0.06433600187301636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,4096,0.1521194616953532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,512,0.06928533713022868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,512,0.045850666364034016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,3584,0.13142293294270832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,256,0.02901653250058492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,3072,0.11293760140736897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,256,0.03139093319574992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,128,0.01832746664683024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,128,0.027294933795928955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,2560,0.09645333290100097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,64,0.01515733301639557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,64,0.02616320053736369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,51200,32,0.013595733046531677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,51200,32,0.026608000199000042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,2048,0.07772586345672608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,1536,0.05726186831792196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,65536,0.4357077280680339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,65536,0.7582143783569336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,768,0.03282986680666606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,51200,0.34589014053344724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,51200,0.5995925267537434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,65536,0.7156255722045899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,16384,0.21223893165588378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,16384,0.1395967960357666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,16384,0.1836693286895752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,512,0.027502934137980144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,12288,0.09943892955780029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,12288,0.13347733815511068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,10240,0.1277184009552002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,256,0.021579732497533165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,128,0.01753386656443278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,8192,0.10202666918436687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,8192,0.07250879605611166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,7168,0.09002026716868082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,8192,0.09719893137613932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,7168,0.06484479904174804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,6144,0.0791978677113851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,7168,0.08577173550923665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,6144,0.05985813140869141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,5120,0.06738879680633544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,5120,0.05189760128657023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,4096,0.054876800378163657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,4096,0.04560533364613851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,12288,0.1500373363494873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,3584,0.05558079878489176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,10240,0.08458453019460042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,3584,0.04178453286488851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,3072,0.043434667587280276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,3584,0.04453973372777303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,3072,0.03837759892145793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,3072,0.039928531646728514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,2560,0.03747946818669637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,2560,0.034935466448465985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,2560,0.033727999528249106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,2048,0.030954666932423908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,2048,0.030789333581924438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,1536,0.02505600055058797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,1536,0.02610986630121867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,1536,0.02213333249092102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,1024,0.01755839983622233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,1024,0.02253226637840271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,51200,1024,0.04475413163503011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,1024,0.01745599905649821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,768,0.01359999974568685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,768,0.02132693330446879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,768,0.01511360009511312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,512,0.010267733534177145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,512,0.01973653237024943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,10240,0.11492693424224854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,256,0.007837866743405659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,256,0.017334399620691936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,256,0.010693333546320597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,128,0.00560746689637502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,128,0.016852267583211265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,128,0.00985599954922994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,64,0.004966400067011515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,64,0.01736533244450887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,16384,32,0.005421866476535797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,4096,0.04955093463261922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,16384,32,0.017809067169825235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,65536,0.3663381258646647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,65536,0.6072021484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,2048,0.02799573342005412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,51200,0.47813119888305666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,65536,0.5947615941365559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,51200,0.268231455485026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,16384,0.09974933465321859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,16384,0.14872852961222333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,12288,0.11366826693216961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,12288,0.08107733726501465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,512,0.013529599706331889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,10240,0.09554986953735352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,10240,0.0689674695332845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,8192,0.07703466415405273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,8192,0.058961065610249844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,8192,0.08788053194681802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,7168,0.06831573645273845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,7168,0.05401386817296346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,51200,0.5471946716308593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,6144,0.06922986507415771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,6144,0.04960106611251831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,6144,0.06437120040257772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,5120,0.051134932041168216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,5120,0.04392640193303426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,5120,0.061465601126352944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,4096,0.042238934834798174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,4096,0.038867199420928956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,16384,0.15672532717386883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,3584,0.037834668159484865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,51200,0.4706250508626302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,12288,0.11640746593475342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,3584,0.036073601245880126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,3072,0.03376533190409343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,3072,0.0328981339931488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,2560,0.02903040051460266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,2560,0.029575467109680176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,10240,0.10110399723052979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,2560,0.029718399047851562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,2048,0.02407360076904297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,16384,6144,0.0727509339650472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,2048,0.026263467470804852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,1536,0.01928000052769979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,1536,0.02342826724052429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,1024,0.013076266646385193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,1536,0.01949333349863688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,1024,0.020768000682195028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,768,0.010883200168609618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,768,0.020317866404851278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,7168,0.07756373087565104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,512,0.00864746669928233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,512,0.017427200078964235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,5120,0.05469439824422201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,256,0.005745066702365876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,256,0.016897066434224447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,4096,0.044691201051076254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,128,0.0047647997736930845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,128,0.016083199779192606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,3584,0.03972800175348918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,3072,0.0344650665918986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,64,0.004462933540344239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,12288,32,0.004692266881465912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,64,0.016660267114639284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,12288,32,0.01686506668726603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,2048,0.025703465938568114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,1024,0.015287466843922935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,768,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,65536,0.322546132405599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,65536,0.5030453364054362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,512,0.011642666657765706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,16384,0.15017706553141277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,51200,0.4032458623250325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,65536,0.5530293146769206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,16384,0.08997440338134766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,51200,0.2569194634755453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,12288,0.10461973349253337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,12288,0.07312106291453044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,10240,0.08503999710083007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,10240,0.06310293277104696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,8192,0.07460052967071533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,8192,0.05368426640828451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,7168,0.060583468278249106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,256,0.0086709330479304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,7168,0.050147199630737306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,12288,128,0.007835733393828075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,6144,0.05544000069300333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,6144,0.04529279867808024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,16384,0.14665385882059734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,12288,0.10730026563008625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,51200,0.4345408121744792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,5120,0.045273598035176596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,4096,0.036883199214935304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,5120,0.05161173343658447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,10240,0.09486506779988607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,4096,0.03545386791229248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,3584,0.034891732533772785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,8192,0.08131093184153239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,3072,0.029539199670155843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,3584,0.03408533334732056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,3072,0.033115732669830325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,3072,0.03083840012550354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,2560,0.026358399788538617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,2560,0.02795093258221944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,2048,0.02177493373552958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,2048,0.025730133056640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,7168,0.0724992036819458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,1536,0.017717333634694417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,1536,0.023206400871276855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,1024,0.012915199995040894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,1024,0.02078933318456014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,6144,0.061887999375661217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,768,0.010786133011182149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,5120,0.04002666473388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,768,0.01989226738611857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,768,0.012409599622090657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,512,0.008481066425641377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,512,0.017218132813771568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,256,0.005705599983533224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,256,0.01647040049235026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,512,0.009943466385205586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,128,0.004846933484077454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,128,0.015979733069737753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,4096,0.041954131921132405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,64,0.00421973317861557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,3584,0.037223466237386066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,64,0.01605226695537567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,10240,32,0.004423466821511587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,10240,32,0.016196266810099284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,2560,0.02835200031598409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,2048,0.023510400454203287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,1536,0.018294399976730345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,65536,0.40184958775838214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,65536,0.26159146626790364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,65536,0.36898771921793616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,1024,0.014362667004267374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,16384,0.09869759877522787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,51200,0.2058272043863932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,12288,0.07683200041453044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,16384,0.07526506582895914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,16384,0.09556266466776529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,12288,0.060836267471313474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,10240,0.06474666595458985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,10240,0.05327786604563395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,10240,0.059621334075927734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,8192,0.052535466353098546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,8192,0.045442132155100505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,7168,0.04670826594034831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,7168,0.04267839988072713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,256,0.007743999858697255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,6144,0.04140160083770752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,6144,0.038446935017903645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,5120,0.03538453181584676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,5120,0.03432106574376424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,51200,0.3173130671183268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,4096,0.032986666758855185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,4096,0.030894933144251506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,4096,0.027889066934585573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,3584,0.026243199904759724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,3584,0.028442666927973433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,10240,128,0.0070261334379514056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,3072,0.02341653307278951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,3072,0.02632213234901428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,3072,0.022980266809463502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,2560,0.02016426722208659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,2560,0.02474773327509562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,12288,0.07129066785176595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,51200,0.286136531829834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,8192,0.050828798611958825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,2048,0.023797333240509033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,1536,0.012795733412106833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,7168,0.045007999738057455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,1536,0.02116159995396932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,1536,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,1024,0.009789866209030152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,1024,0.019322667519251505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,6144,0.03925439914067586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,768,0.008373333017031352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,768,0.017067732413609822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,768,0.010574932893117268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,512,0.006844800213972728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,512,0.016845866044362386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,512,0.008711466193199157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,256,0.005107200145721436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,256,0.016364799936612447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,256,0.007482666770617168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,128,0.004401066899299621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,128,0.01585493286450704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,2048,0.016578132907549538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,64,0.0039658665657043455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,64,0.016103466351826988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,2048,0.01699413259824117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,8192,32,0.0044383997718493145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,8192,32,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,3584,0.02544533411661784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,1024,0.011688533425331115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,5120,0.03404479821523031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,65536,0.2330143928527832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,65536,0.36480639775594076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,2560,0.01958613395690918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,51200,0.28974294662475586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,16384,0.10276587009429931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,51200,0.18649706840515137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,16384,0.06938772996266683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,12288,0.06975680192311605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,51200,0.2643967946370443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,12288,0.05749760071436564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,12288,0.06776320139567057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,10240,0.05857280095418295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,8192,0.047115735212961835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,10240,0.05045226812362671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,8192,0.04294613202412923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,7168,0.04255040089289348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,7168,0.04018346468607585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,6144,0.03782080014546712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,6144,0.03675839900970459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,5120,0.03210666577021281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,8192,128,0.0069567998250325514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,5120,0.03327893416086833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,4096,0.026235733429590864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,4096,0.028510934114456175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,65536,0.3336064020792643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,3584,0.027116799354553224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,3584,0.026859732468922932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,10240,0.05682666699091593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,16384,0.09233386516571045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,3072,0.021486934026082358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,3072,0.025441066424051924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,3072,0.021307732661565146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,2560,0.018311466773351034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,8192,0.0482037345568339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,7168,0.042277332146962485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,2048,0.014910933375358582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,2048,0.015222400426864624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,1536,0.011947733163833619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,6144,0.03694186607996623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,1536,0.020350933074951172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,1536,0.013011200229326883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,1024,0.009190400441487629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,1024,0.010777599612871806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,1024,0.019217065970102944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,768,0.007881600161393483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,768,0.017189333836237587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,512,0.006351999938488007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,512,0.016953599452972413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,512,0.0077567999561627705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,256,0.004890666902065277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,256,0.016782933473587038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,256,0.0065653334061304735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,128,0.004676266511281332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,128,0.016772266228993735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,2560,0.023539199431737264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,5120,0.031727999448776245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,2048,0.0221781333287557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,64,0.004669866462548574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,64,0.016938666502634682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,7168,32,0.004937600096066793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,7168,32,0.01665493349234263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,4096,0.02625280022621155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,65536,0.20864747365315756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,65536,0.3150218645731608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,65536,0.3106005350748698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,51200,0.16711254119873048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,16384,0.08616533279418945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,16384,0.06511360009511312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,16384,0.08413440386454264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,51200,0.24734400113423666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,12288,0.053997866312662755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,12288,0.06929600238800049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,10240,0.04680853287378947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,10240,0.06945172945658365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,8192,0.042819201946258545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,8192,0.04007680018742879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,3584,0.023639466365178427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,7168,0.03877439896265666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,7168,0.038031999270121256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,2560,0.018438400824864705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,7168,0.03988800048828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,6144,0.03415466547012329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,6144,0.03493119875590007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,6144,0.03527466853459676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,5120,0.02921066681543986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,5120,0.029845333099365233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,51200,0.25063467025756836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,768,0.009252267082532246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,4096,0.025678932666778564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,4096,0.026818132400512694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,3584,0.022027732928593953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,3584,0.025701334079106648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,7168,128,0.006029866635799408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,3072,0.01944640080134074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,3072,0.024272000789642333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,5120,0.030962133407592775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,2560,0.01625920037428538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,2560,0.02328959902127584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,12288,0.06417920192082724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,10240,0.05303039948145548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,2048,0.013335466384887695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,2048,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,1536,0.011198932925860088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,8192,0.04680213530858358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,1536,0.0206389327843984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,1024,0.00881813367207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,1024,0.018278400103251137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,768,0.007778133451938629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,768,0.01721386710802714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,4096,0.025217066208521526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,768,0.008842666943868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,512,0.005876266459623972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,512,0.017257599035898845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,256,0.004679466784000397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,3584,0.022756266593933105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,256,0.006372266511122386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,256,0.017116800944010416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,128,0.0047989333669344585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,128,0.016148266196250916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,128,0.005958400170008342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,2048,0.021705599625905354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,64,0.004349866509437561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,6144,32,0.004445866743723551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,64,0.016129066546758018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,3072,0.020525866746902467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,65536,0.2550410588582357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,65536,0.1886026700337728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,51200,0.2040010611216227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,51200,0.14954880078633626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,2560,0.01755946675936381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,16384,0.06717546780904135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,51200,0.22665279706319175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,16384,0.08147733211517334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,12288,0.05306346813837687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,12288,0.04907199939092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,10240,0.04420906702677409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,12288,0.058881068229675294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,10240,0.04337066809336344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,8192,0.04188906749089559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,6144,32,0.016385066509246825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,8192,0.037752532958984376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,7168,0.0357258677482605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,7168,0.03504746754964193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,1536,0.01266986628373464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,6144,0.028857600688934327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,6144,0.03249386747678121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,6144,0.03367786804835002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,1024,0.010263466835021972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,5120,0.02802666624387105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,5120,0.029167999823888142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,5120,0.028489599625269573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,4096,0.020489599307378134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,4096,0.026526933908462523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,3584,0.020173867543538414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,16384,0.0618346651395162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,3584,0.02486613392829895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,3072,0.01811520059903463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,3584,0.0211136003335317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,3072,0.023835732539494833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,2560,0.014071466525395713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,6144,512,0.007747200131416321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,2560,0.022636799017588298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,2048,0.013171199957529703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,2560,0.01643519997596741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,2048,0.02141973376274109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,1536,0.00951573352018992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,1536,0.019617066780726115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,1536,0.011716266473134358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,1024,0.008474666873613994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,1024,0.018092799186706542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,1024,0.009831466277440389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,768,0.006614399949709575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,768,0.016603733102480568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,768,0.00832533339659373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,512,0.00505920002857844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,512,0.01630826691786448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,65536,0.28485546112060545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,512,0.007386666536331177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,4096,0.024091732501983643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,256,0.015784533818562825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,256,0.004439466694990794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,256,0.006053333481152853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,128,0.0038773333032925926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,128,0.01541759967803955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,3072,0.019049600760142008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,64,0.015629866719245912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,32,0.003705599904060364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,5120,32,0.0156768004099528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,65536,0.21098987261454263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,10240,0.04955306847890218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,8192,0.042149333159128825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,65536,0.16448853810628256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,7168,0.03779840071996053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,51200,0.16782399813334148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,51200,0.12798826694488524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,16384,0.059935998916625974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,16384,0.056081068515777585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,16384,0.06884053548177084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,5120,64,0.0035135999321937563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,12288,0.05149120092391968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,10240,0.043333331743876144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,12288,0.052786131699879967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,10240,0.04055146773656209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,8192,0.03552746772766113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,8192,0.035861333211263016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,8192,0.03792320092519124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,7168,0.03158400058746338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,7168,0.03268479903539022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,2048,0.01393280029296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,7168,0.03415040175120036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,6144,0.027774933973948163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,6144,0.029956267277399702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,6144,0.02994026740392049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,5120,0.023836799462636313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,5120,0.028357332944869994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,4096,0.019741866985956827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,4096,0.025166932741800947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,12288,0.04596586624781291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,3584,0.017564799388249716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,5120,128,0.00552106648683548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,3584,0.023912533124287923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,3072,0.015590399503707886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,3072,0.02262079914410909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,3072,0.01771199901898702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,2560,0.013459199666976928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,2560,0.021549866596857706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,2048,0.01148373285929362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,2048,0.020142932732899986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,65536,0.263209597269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,1536,0.009435733159383137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,1536,0.0186463991800944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,51200,0.20889813105265298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,1024,0.007530666887760162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,1024,0.016908800601959227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,10240,0.04506346782048543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,768,0.006296533346176148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,768,0.016375466187795003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,5120,0.025577600797017413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,512,0.0051018665234247845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,512,0.01618773341178894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,4096,0.021334399779637657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,256,0.004020266731580098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,256,0.015611732999483744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,3584,0.019476266702016194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,128,0.0036138666172822317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,128,0.015494400262832641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,2560,0.015425067146619162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,64,0.003790933390458425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,64,0.015594666202863058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,4096,32,0.0038304001092910765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,2048,0.0132533331712087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,65536,0.20110507011413575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,65536,0.15030186971028645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,1536,0.011252267162005107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,51200,0.14741759300231932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,1024,0.008919466535250347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,128,0.005840000013510386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,51200,0.12303893566131592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,16384,0.05378559827804565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,16384,0.048981332778930665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,768,0.008400000135103862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,12288,0.038865065574645995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,51200,0.2086634635925293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,12288,0.04044160048166911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,4096,32,0.015794133146603904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,10240,0.03560213247934978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,8192,0.026828799645106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,8192,0.03115839958190918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,512,0.007368533313274384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,7168,0.024360533555348715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,7168,0.02837973237037659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,4096,256,0.006297599772612255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,6144,0.021846399704615275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,6144,0.02610879937807719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,10240,0.032788266738255814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,5120,0.0183786670366923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,5120,0.024194133281707764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,65536,0.24955199559529623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,4096,0.01671573321024577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,4096,0.022024534145991006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,16384,0.07212693691253662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,12288,0.05315946737925211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,3584,0.013380266229311624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,3584,0.022055466969807945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,10240,0.04565759897232056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,3072,0.011914666493733723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,3072,0.02106346686681112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,8192,0.03980693419774373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,2560,0.010551466544469198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,2560,0.02044693430264791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,2560,0.015156267086664834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,2048,0.008936533331871032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,2048,0.019118932882944743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,7168,0.03527573347091675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,1536,0.007877333462238312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,1536,0.017808000246683754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,6144,0.030089600880940752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,1024,0.005764266848564148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,1024,0.017178666591644288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,5120,0.025242666403452556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,768,0.005048533280690512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,768,0.01657919983069102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,4096,0.021631999810536703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,512,0.004429866870244345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,512,0.01611199975013733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,3584,0.01981546680132548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,256,0.004105599969625473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,3072,0.01717653274536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,256,0.016267733772595723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,256,0.005871999760468801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,128,0.004030933231115341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,128,0.015413332978884378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,64,0.0036789332826932275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,64,0.01543786625067393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3584,32,0.0037450666228930154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,2048,0.013006933530171714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3584,32,0.015731199582417806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,65536,0.15853226979573568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,65536,0.13722559611002605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,51200,0.1271018664042155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,51200,0.10986453692118328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,1536,0.010492799679438274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,16384,0.04949440161387126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,16384,0.04691733519236247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,1024,0.00864746669928233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,51200,0.1921610673268636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,16384,0.06919360160827637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,12288,0.05488106807072958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,12288,0.038636799653371176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,10240,0.033861335118611655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,10240,0.043755733966827394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,8192,0.025015467405319215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,8192,0.029154133796691895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,8192,0.037628801663716634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,7168,0.022798933585484824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,7168,0.02725013295809428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,6144,0.01998400092124939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,7168,0.03344746828079224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,6144,0.025490132967631023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,5120,0.01681493322054545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,5120,0.024105600516001382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,10240,0.04850773413976033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,4096,0.013365333278973898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,4096,0.021897600094477335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,512,0.006846933563550313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,4096,0.021272534132003786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,3584,0.012492799758911132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,3584,0.021177599827448525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,3072,0.01114453375339508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,768,0.007737599809964498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,3072,0.020812799533208214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,2560,0.010010666648546855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,2560,0.019402666886647543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3584,128,0.005329066514968872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,2048,0.00869653324286143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,2048,0.01813653310139974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,65536,0.24187307357788085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,1536,0.007549866537253062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,1536,0.017356799046198527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,12288,0.051692799727121985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,1024,0.005640533566474914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,1024,0.01641386648019155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,1024,0.008276266853014629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,768,0.005003733436266581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,768,0.017010132471720375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,6144,0.02914773424466451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,512,0.004387199878692627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,512,0.016498133540153503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,5120,0.025542400280634564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,256,0.0043381333351135256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,256,0.01614720026652018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,3584,0.01861226757367452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,128,0.003732266773780187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,128,0.015293866395950317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,3072,0.016698666413625083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,64,0.0033919999996821085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,2560,0.014596266547838846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,3072,32,0.003537066777547201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,64,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,2048,0.012113066514333089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,3072,32,0.015161599715550741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,65536,0.14077226320902508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,65536,0.12483946482340495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,51200,0.11287786960601806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,51200,0.10066346327463786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,16384,0.04180266857147217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,1536,0.010024533669153849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,16384,0.04561813275019328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,768,0.007354666789372762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,12288,0.03916906515757243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,12288,0.03740799824396769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,12288,0.050678400198618564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,512,0.006540800134340922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,10240,0.03239253362019857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,10240,0.04260480006535848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,8192,0.026452267169952394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,10240,0.03234346707661946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,8192,0.02786986629168193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,7168,0.021575466791788737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,256,0.005553066730499268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,7168,0.02678826649983724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,6144,0.01923946738243103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,6144,0.024846933285395303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,3072,128,0.005151999990145365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,5120,0.018408532937367758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,5120,0.023729066054026283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,65536,0.23162986437479655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,4096,0.015133866667747497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,4096,0.021993599335352578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,51200,0.18898986180623373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,3584,0.013264000415802002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,3584,0.021118932962417604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,16384,0.06531733274459839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,3072,0.010003200173377991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,3072,0.02025173306465149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,3072,0.01611733337243398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,2560,0.008719999591509502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,2560,0.020148267348607383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,2048,0.008869333068529765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,2048,0.0188426673412323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,2048,0.011913599570592244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,1536,0.0070709332823753355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,1536,0.01768853267033895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,8192,0.03621973196665446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,1024,0.0058442667126655575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,1024,0.016509866714477538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,7168,0.03290026585261027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,1024,0.00827946662902832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,768,0.0050240000089009605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,768,0.016148266196250916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,768,0.007393066585063934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,512,0.004404266675313314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,6144,0.02832213242848714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,512,0.016025599837303162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,512,0.006467199822266896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,256,0.0037503999968369803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,128,0.0033429334561030067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,256,0.005666133264700572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,256,0.01572053333123525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,64,0.00310506671667099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,128,0.014914133151372275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2560,32,0.0031946666538715364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,64,0.0149536003669103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2560,32,0.015024000406265258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,65536,0.10901333491007488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,65536,0.111134934425354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,5120,0.024277333418528238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,51200,0.08770346641540527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,65536,0.2213343938191732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,51200,0.08977706432342529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,16384,0.03769493500391642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,16384,0.042182401816050215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,12288,0.02988053361574809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,12288,0.03865706523259481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,4096,0.02074880003929138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,10240,0.02616746624310811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,10240,0.03350720008214315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,3584,0.01805973251660665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,8192,0.022316799561182658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,2560,0.014032000303268432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,8192,0.029755733410517377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,7168,0.024420267343521117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,7168,0.027831466992696126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,1536,0.009705600142478944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,6144,0.022667733828226726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,6144,0.026929066578547163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,6144,0.02566933234532674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,5120,0.020023467143376668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,5120,0.024652800957361855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2560,128,0.0052042668064435325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,4096,0.01518933375676473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,4096,0.022223999102910362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,4096,0.01874026656150818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,3584,0.013569066921869913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,3584,0.021824000279108684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,51200,0.1681653340657552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,3072,0.011582932869593303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,16384,0.059672534465789795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,3072,0.02040106654167175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,2560,0.010243200262387593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,2560,0.019061332941055296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,12288,0.04548693497975667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,2048,0.008873599767684936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,2048,0.018759467204411826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,10240,0.039641598860422775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,1536,0.0076885332663853955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,8192,0.03277759949366252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,1536,0.01806933283805847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,1536,0.00944640040397644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,1024,0.0061152001221974695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,7168,0.029848533868789672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,1024,0.01687039931615194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,768,0.005355733136336008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,768,0.01628159979979197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,5120,0.02178879976272583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,512,0.0046165332198143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,512,0.015828266739845276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,1024,0.00790719985961914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,256,0.0038133333126703895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,256,0.015465600291887918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,3584,0.016780799627304076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,128,0.003409066547950109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,128,0.01511679987112681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,128,0.005186133086681366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,64,0.003221333275238673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,64,0.015074132879575094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,2048,32,0.0032874666154384612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,3072,0.015013333161671957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,2048,32,0.015222400426864624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,2560,0.012459733088811238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,65536,0.08889919916788737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,51200,0.07244799931844076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,65536,0.10196159680684407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,2048,0.010616532961527507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,51200,0.08043413162231446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,16384,0.029625600576400755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,51200,0.17177492777506512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,16384,0.04259200096130371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,12288,0.0320906658967336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,12288,0.03462719917297363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,10240,0.026927999655405682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,10240,0.03859093189239502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,10240,0.030943999687830608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,8192,0.02215786576271057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,8192,0.027800534168879194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,7168,0.01983573238054911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,7168,0.026158932844797773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,7168,0.02984106739362081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,6144,0.0179584006468455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,6144,0.025088000297546386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,768,0.0071946665644645694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,5120,0.015282133221626281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,5120,0.02329813241958618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,512,0.006577066580454509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,4096,0.013249066472053529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,4096,0.021977599461873373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,4096,0.018218666315078735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,3584,0.011839999755223592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,3584,0.02077546715736389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,2048,256,0.005565866827964783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,3072,0.010801066954930622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,3072,0.019832533597946168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,65536,0.2120533307393392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,2560,0.009665066997210186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,16384,0.062084265549977626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,12288,0.045658667882283524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,2048,0.007881600161393483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,2048,0.017773866653442383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,8192,0.034380801518758136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,1536,0.0065184002121289565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,1536,0.01710933248202006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,1536,0.009309867024421692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,1024,0.005182933310667674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,1024,0.016294399897257485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,6144,0.026374399662017822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,768,0.004543999830881754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,768,0.01609386702378591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,5120,0.022733867168426514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,512,0.003988266736268997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,512,0.015583999951680503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,512,0.006543999910354615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,256,0.0034634667138258614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,256,0.0150709331035614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,2560,0.018780799706776936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,128,0.0032117334504922234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,128,0.01479039986928304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,3584,0.016552533706029257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,64,0.0030602666238943735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,64,0.014693333705266317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1536,32,0.003127466638882955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1536,32,0.014885333180427552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,65536,0.06293226480484009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,3072,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,65536,0.08636159896850586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,51200,0.05178986787796021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,51200,0.06951573689778646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,2560,0.012711466352144877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,16384,0.02137706677118937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,16384,0.035588268438975015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,2048,0.01106773316860199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,12288,0.016829866170883178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,12288,0.030551467339197797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,12288,0.04173760016759236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,1024,0.007846400141716003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,10240,0.015065600474675497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,10240,0.028550400336583452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,768,0.007065600156784058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,8192,0.016247466206550598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,8192,0.025303467114766436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,256,0.005481599768002828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,7168,0.015384533007939658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,7168,0.02445866664250692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,7168,0.026999467611312868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1536,128,0.005067733426888784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,6144,0.015441067020098367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,6144,0.023018666108449302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,5120,0.011890133221944172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,5120,0.021423999468485513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,5120,0.020205867290496827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,4096,0.010133333007494609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,4096,0.02067520022392273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,4096,0.017053866386413576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,3584,0.009431466460227966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,3584,0.01903466582298279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,65536,0.1916096051534017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,3072,0.008794666330019633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,3072,0.0187008003393809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,51200,0.15272000630696614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,2560,0.008667733271916707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,2560,0.018169599771499633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,16384,0.05484266678492228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,2048,0.006520533561706543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,2048,0.017326933145523072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,2048,0.009966933727264404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,1536,0.005643733342488607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,1536,0.016965333620707193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,1536,0.009081600109736125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,10240,0.035813331604003906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,1024,0.016214399536450704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,768,0.0042581334710121155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,768,0.01560640037059784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,8192,0.030564266443252563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,512,0.0038794666528701783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,512,0.015333333611488342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,6144,0.02328426639238993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,256,0.003340800106525421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,256,0.015075199802716575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,1024,0.004833066463470459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,128,0.0031328000128269195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,3584,0.015575466553370157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,128,0.01470080018043518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,64,0.002926933268706004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,64,0.014858667055765787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,1024,32,0.0029919999341169994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,1024,32,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,65536,0.05254720052083334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,65536,0.08116373221079508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,2560,0.011743999520937602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,51200,0.04323840141296387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,51200,0.06432000001271566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,128,0.004875733455022176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,16384,0.016314666469891867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,16384,0.03137173255284627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,1024,0.0077248002092043565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,3072,0.013649066289265951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,768,0.00695360004901886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,12288,0.0335370659828186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,12288,0.03471893469492594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,10240,0.016247466206550598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,10240,0.029963733752568562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,8192,0.013619200388590495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,8192,0.026985599597295122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,512,0.006485333542029063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,7168,0.012486400206883748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,7168,0.025473066171010333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,7168,0.022550400098164877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,1024,256,0.005300266544024149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,6144,0.017079466581344606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,6144,0.02314773400624593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,5120,0.015153066317240397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,5120,0.022089600563049316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,12288,0.01769599914550781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,4096,0.01306773324807485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,4096,0.02068480054537455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,65536,0.16475626627604167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,3584,0.010378666718800863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,3584,0.02032426595687866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,3584,0.013062399625778199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,3072,0.009979733824729919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,51200,0.1334112008412679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,3072,0.019099734226862588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,2560,0.008752000331878663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,3072,0.011517866452534994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,2560,0.018411733706792197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,2048,0.007563733557860057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,2048,0.01767573356628418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,2048,0.009085866808891296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,1536,0.00625600020090739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,1536,0.016976000865300496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,16384,0.04531306823094686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,1024,0.004886400202910105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,1024,0.01649066706498464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,1024,0.006753066678841908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,768,0.004593066871166229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,10240,0.029445334275563555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,768,0.015596800049146018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,8192,0.025510400533676147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,512,0.003881600002447764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,512,0.015390933553377787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,6144,0.019730132818222047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,256,0.0033301333586374915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,256,0.01523413360118866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,5120,0.017193599541982015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,128,0.003160533308982849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,128,0.014729600151379904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,4096,0.014429866274197897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,64,0.0028586665789286296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,64,0.014629333217938741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,768,32,0.0029386666913827257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,768,32,0.01474666694800059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,65536,0.03887999852498372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,65536,0.07301119963328043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,65536,0.1277407964070638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,51200,0.032314666112263996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,51200,0.05857173204421997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,51200,0.10210026899973552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,2560,0.010406399766604107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,16384,0.01663146714369456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,16384,0.034484267234802246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,16384,0.02985386649767558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,12288,0.013097600142161051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,12288,0.027240532636642455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,12288,0.025986133019129436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,10240,0.011332266529401143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,10240,0.023564799626668295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,1536,0.007949866851170858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,8192,0.012325333555539449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,8192,0.018161066373189292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,8192,0.022472532590230306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,7168,0.01156160036722819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,7168,0.01632533371448517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,7168,0.021491199731826782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,6144,0.010620799660682679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,6144,0.021041067441304524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,5120,0.009404800335566203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,768,0.0065760001540184024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,5120,0.020933334032694498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,5120,0.013140267133712769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,4096,0.00832533339659373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,4096,0.019341866175333657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,4096,0.011346133550008138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,3584,0.007851733267307282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,3584,0.0107914666334788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,3584,0.018797866503397622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,3072,0.0071370666225751235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,2560,0.006435200075308482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,3072,0.01784106691678365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,2560,0.017409066359202065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,2048,0.006450133522351582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,512,0.005971199770768484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,2048,0.016753067572911583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,1536,0.0050357331832249965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,1536,0.016315733393033348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,256,0.005236266553401947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,1024,0.0043136000633239744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,768,128,0.004999466737111409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,1024,0.006295466423034668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,1024,0.015948800245920818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,768,0.004036266605059306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,768,0.015500799814860026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,768,0.0061365331212679545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,512,0.003684266656637192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,512,0.015338666240374246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,512,0.005478399991989136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,256,0.003302400062481562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,256,0.015382400155067444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,128,0.0029546665648619336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,128,0.014792533715566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,128,0.004748799900213877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,64,0.0030293333033720653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,64,0.01472106675306956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,512,32,0.0028319999575614927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,512,32,0.014840533336003622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,65536,0.028424533208211263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,65536,0.06775680383046469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,10240,0.023202133178710938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,51200,0.02322559952735901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,51200,0.05326506694157919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,51200,0.08164052963256836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,16384,0.00983893374602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,6144,0.014778666694959006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,16384,0.027156267563501996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,16384,0.025214932362238568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,12288,0.010945066809654236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,12288,0.022657066583633423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,10240,0.009090133508046468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,10240,0.023702400922775268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,3072,0.009525332848230999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,8192,0.008183466891447704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,2560,0.008779733379681905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,8192,0.021691733598709108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,8192,0.015685333808263143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,7168,0.007602133353551229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,7168,0.021181867520014445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,7168,0.014392532904942832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,2048,0.007804800073305767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,6144,0.007044266661008198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,6144,0.019827200969060262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,5120,0.00795839975277583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,1536,0.007311999797821045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,5120,0.019874133666356406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,5120,0.011326932907104492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,4096,0.007985066870848339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,4096,0.018989866971969603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,4096,0.010487467050552368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,3584,0.006717866659164429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,3584,0.01875093380610148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,3584,0.010019200046857198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,3072,0.006424533327420552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,3072,0.017910399039586387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,3072,0.008913066983222962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,2560,0.006175999840100607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,2560,0.017452800273895265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,512,256,0.0050240000089009605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,2560,0.00808426688114802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,2048,0.005492266515890757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,1536,0.004834133386611939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,2048,0.016909867525100708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,1536,0.016562133034070333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,1024,0.004182399809360504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,1024,0.015650133291880287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,1024,0.005990399916966756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,768,0.0037578667203585305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,10240,0.018606932957967122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,65536,0.1042954683303833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,6144,0.012900267044703165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,512,0.003492266684770584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,512,0.005456000069777171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,512,0.0151146670182546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,256,0.0030933332939942675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,256,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,128,0.0029056000212828318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,128,0.014668800433476768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,128,0.004700799783070883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,64,0.002834133307139079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,64,0.014503467082977294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,256,32,0.0028661333024501802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,32,0.014546133081118264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,65536,0.020374399423599244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,65536,0.06184106667836507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,256,768,0.015410133202870688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,51200,0.013318399588267008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,12288,0.021747199694315593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,51200,0.04889813264211019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,51200,0.0793013334274292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,16384,0.0095360000928243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,16384,0.023333332935969033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,12288,0.009097599983215332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,12288,0.022500266631444298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,12288,0.021596799294153847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,10240,0.008186666667461396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,10240,0.021296000480651854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,10240,0.01881493330001831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,8192,0.007188266515731812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,8192,0.02050666610399882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,1536,0.00694400022427241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,7168,0.006681600213050842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,7168,0.020454400777816774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,768,0.005676800012588501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,6144,0.006260266900062561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,6144,0.019667200247446694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,6144,0.01276693344116211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,5120,0.006563200056552887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,5120,0.020191999276479085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,5120,0.011443199714024861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,256,0.004904533425966898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,4096,0.006101333101590474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,4096,0.010137599706649781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,4096,0.01880106727282206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,3584,0.00653546651204427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,3584,0.018573866287867228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,3072,0.00621013343334198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,3072,0.017760000626246133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,3072,0.008734933535257975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,2560,0.0061247999469439185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,2560,0.01758400003115336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,2560,0.007987200220425924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,256,2048,0.0072543998559316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,2048,0.005392000079154968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,1536,0.004771199822425842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,1536,0.016244266430536905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,1536,0.006971733272075653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,1024,0.004057600100835165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,1024,0.015564800302187601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,65536,0.10161813100179036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,1024,0.005996799965699514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,768,0.0037589333951473236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,768,0.015238400300343832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,768,0.005602133274078369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,512,0.003442133218050003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,512,0.015178666512171427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,256,0.003019733230272929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,256,0.014909866452217101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,256,0.004905599852403005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,16384,0.02740266720453898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,128,0.002828799933195114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,128,0.014578133821487427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,128,0.0047978664437929785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,64,0.002757333219051361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,64,0.014447999993960061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,128,32,0.002755200117826462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,3584,0.009802666306495667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,32,0.014428800344467163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,65536,0.013964800039927163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,51200,0.012007466952006022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,65536,0.058516267935434976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,16384,0.008243200182914735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,51200,0.04449706474939982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,128,2048,0.01686613361040751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,8192,0.015736533204714458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,12288,0.0072405333320299785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,16384,0.024226133028666177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,10240,0.00681386689345042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,8192,0.006434133152167003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,12288,0.021614933013916017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,10240,0.02071040074030558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,6144,0.005974400043487549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,8192,0.019355734189351402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,7168,0.02022613286972046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,5120,0.0063178668419520065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,6144,0.020258132616678873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,4096,0.005993600189685822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,5120,0.01951040029525757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,3584,0.006444799900054932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,4096,0.01866453289985657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,3072,0.006129066646099091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,3584,0.018516266345977785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,7168,0.014371200402577718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,2560,0.006097066899140676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,3072,0.017875200510025023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,2048,0.005471999943256378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,2560,0.01755626598993937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,1536,0.00468800018231074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,2048,0.01676373283068339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,1024,0.004031999905904134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,1536,0.0164192001024882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,768,0.003714133302370707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,1024,0.015621333320935567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,512,0.0034634667138258614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,768,0.015522133310635886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,256,0.0030250666042168934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,512,0.015241600076357522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,128,0.002880000074704488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,256,0.01471466620763143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,64,0.0027146667242050173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,128,0.01440000037352244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,32,0.002792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,64,0.014545067151387533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,65536,0.01262399951616923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,64,32,0.014417066176732381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,51200,0.010627200206120808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,65536,0.05579200188318888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,16384,0.008969600001970928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,51200,0.04147839943567912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,12288,0.00746666689713796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,16384,0.02367253303527832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,10240,0.007000533243020375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,12288,0.021129600207010903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,8192,0.006515199939409892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,7168,0.006303999821345012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,10240,0.021307732661565146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,8192,0.02108373244603475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,7168,0.020108799139658608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,6144,0.005996799965699514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,5120,0.0062826668222745255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,4096,0.0059456000725428265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,6144,0.019418666760126747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,5120,0.019796266158421835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,3584,0.006359466910362243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,4096,0.01857066750526428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,3072,0.006044800082842508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,3584,0.018053332964579262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,2560,0.005946666498978933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,3072,0.017566933234532674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,2048,0.005399466554323832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,2560,0.017525333166122436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,1536,0.004648533463478088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,2048,0.01690666675567627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,1024,0.003987200061480204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,1536,0.016482133666674295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,768,0.00360000009338061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,1024,0.015688533584276833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,512,0.0033674667278925574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,768,0.015405866503715514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,512,0.015125333269437154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,128,0.00279573326309522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,256,0.015056000153223673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,64,0.0026528000831604003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,64,7168,0.00625600020090739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,128,0.014444800217946372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,32,0.0027061333258946735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,64,0.014630400141080222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,193,32,32,0.01442026694615682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,512,0.005357866485913595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,16384,0.7741088231404623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,193,128,2048,0.007144533097743988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,193,32,256,0.002961066613594691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,16384,0.4187061309814453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,12288,0.3221824010213216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,12288,0.5926752090454102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,51200,1.245895512898763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,12288,0.5230688095092774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,10240,0.2840245246887207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,10240,0.4965173403422038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,8192,0.22312000592549644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,8192,0.3946645418802897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,10240,0.4575392087300618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,51200,2.420342508951823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,8192,0.3625098546346029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,7168,0.2002101262410482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,7168,0.3464319864908854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,6144,0.17605973879496256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,6144,0.3523893356323242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,5120,0.2541013399759928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,6144,0.27348693211873376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,5120,0.15192319552103678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,4096,0.12681492964426677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,4096,0.20673920313517252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,3584,0.18261866569519042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,3072,0.159662930170695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,3072,0.10087253252665203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,3072,0.1349130630493164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,16384,0.7014944076538085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,2560,0.13570027351379393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,2560,0.09873706499735514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,2560,0.1143232027689616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,2048,0.11167360146840413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,2048,0.07539520263671876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,3584,0.11426880359649658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,1536,0.08650026321411133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,1536,0.06470933357874552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,1024,0.06191680034001669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,1024,0.050777598222096765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,1024,0.049252267678578696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,51200,2.2242144266764323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,768,0.049831465880076094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,768,0.046271999677022294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,512,0.037649067242940266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,768,0.05048213402430216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,512,0.03801920016606648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,256,0.02498133381207784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,256,0.03219306667645772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,256,0.030561065673828124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,128,0.019049600760142008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,128,0.028487465778986615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,128,0.027687466144561766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,4096,0.17946133613586426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,5120,0.22764800389607748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,64,0.014461867014567056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,32,0.01502293348312378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,32,0.029075199365615846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,64,0.02783573269844055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,1536,0.06713706652323405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,512,0.03588693141937256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,65536,1.6674196879069012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,65536,2.198271942138672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,65536,3.551844278971354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,16384,0.7203818639119466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,7168,0.3188960075378418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,51200,1.2444853464762369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,51200,2.7375221252441406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,12288,0.5520405451456706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,12288,0.30099414189656576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,16384,0.41440639495849607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,51200,1.8043168385823567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,10240,0.243615992863973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,10240,0.46055145263671876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,8192,0.3925098737080892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,8192,0.198854398727417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,2048,0.09064746697743734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,7168,0.1795178731282552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,7168,0.3078122774759928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,6144,0.26984319686889646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,6144,0.15982826550801593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,3584,0.15560107231140136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,5120,0.22485013008117677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,5120,0.1350272019704183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,4096,0.17846399943033855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,4096,0.11350399653116863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,16384,0.5610207875569662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,3584,0.17223466237386068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,3584,0.10405653317769367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,12288,0.447159481048584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,10240,0.3601674715677897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,3072,0.15411307017008463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,3072,0.0983136018117269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,8192,0.30823678970336915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,2560,0.11733012994130451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,2560,0.08515306313832602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,2048,0.09869866371154785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,7168,0.2694218635559082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,2048,0.066648534933726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,1536,0.07614506880442301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,2048,0.07645653088887533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,1536,0.05655466715494791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,6144,0.22920212745666504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,1024,0.052892800172170004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,1024,0.04533546765645345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,1024,0.04381440083185832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,5120,0.1899232069651286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,768,0.04269866545995076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,768,0.032793599367141726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,768,0.04302933216094971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,512,0.031828266382217404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,512,0.03467413187026978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,256,0.021191465854644775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,256,0.029190399249394733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,256,0.02076586683591207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,128,0.014983466267585755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,128,0.026732800404230754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,128,0.017366399367650352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,64,0.011962667107582092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,4096,0.15222080548604328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,64,0.026423466205596925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,51200,32,0.013685333728790283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,51200,32,0.026106667518615723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,512,0.02707093358039856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,3584,0.13104960123697917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,65536,0.4360639890034993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,65536,0.7533653259277344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,3072,0.1131935993830363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,2560,0.09532586733500162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,16384,0.19512853622436524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,51200,0.345635191599528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,16384,0.12317333221435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,51200,0.6050570805867512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,12288,0.09849279721577962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,12288,0.15113813082377117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,65536,0.6800341288248698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,12288,0.13771732648213703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,51200,1536,0.0586517333984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,10240,0.12559359868367512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,10240,0.08539093335469564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,8192,0.07155306339263916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,8192,0.10246187051137287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,10240,0.11601706345876057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,8192,0.09445546468098959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,7168,0.08944746653238932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,6144,0.07976319789886474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,7168,0.0842688004175822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,6144,0.059417601426442465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,5120,0.06663360198338827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,6144,0.07359146277109782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,5120,0.052729598681131994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,4096,0.055078399181365964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,4096,0.04531626701354981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,4096,0.0507370670636495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,3584,0.04912319978078206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,3584,0.04242453177769979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,3072,0.043305599689483644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,3584,0.04357440074284871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,3072,0.03876800139745076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,2560,0.037020798524220785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,2560,0.03504853248596192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,2048,0.030959999561309813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,2560,0.033817601203918454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,2048,0.0304639995098114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,1536,0.02495253284772237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,1536,0.026374399662017822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,1536,0.022170666853586832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,1024,0.017400532960891724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,1024,0.02241920034090678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,768,0.013608533143997192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,768,0.021331199010213218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,7168,0.0651477336883545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,512,0.010149332880973815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,512,0.020231467485427857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,51200,0.5426005045572917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,256,0.008035199840863545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,256,0.01707306702931722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,256,0.010696533322334289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,5120,0.06095360120137533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,128,0.005636266867319743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,128,0.009760000308354696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,128,0.01688213348388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,64,0.004869333406289419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,32,0.0053610667586326596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,64,0.017044266064961754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,32,0.01772800087928772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,3072,0.03868800004323324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,2048,0.027846399943033857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,768,0.015174399813016257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,65536,0.36545600891113283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,65536,0.6067391713460286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,51200,0.47841386795043944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,16384,0.1967850685119629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,51200,0.2921642621358236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,16384,0.10044906934102375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,12288,0.13448746999104816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,12288,0.08060586452484131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,10240,0.12960320313771564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,10240,0.07056639989217123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,512,0.012910933295885722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,8192,0.11623679796854655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,8192,0.058525868256886805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,65536,0.5829386393229167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,16384,0.15577707290649415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,7168,0.08081066608428955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,12288,0.11829546292622883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,7168,0.054257067044576016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,6144,0.08243520259857177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,6144,0.04883093436559041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,16384,0.17981972694396972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,10240,0.1012447992960612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,5120,0.059036799271901454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,5120,0.0440224011739095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,4096,0.04224853515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,5120,0.05401279926300049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,4096,0.03856000105539958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,3584,0.03781439860661824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,3584,0.03569493293762207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,3584,0.03863893349965413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,3072,0.03332053422927857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,3072,0.03267413377761841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,3072,0.035190399487813315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,51200,0.4699605305989583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,2560,0.03155946731567383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,2560,0.029767467578252153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,2048,0.023923200368881226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,8192,0.08632746537526449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,2048,0.02442986567815145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,1536,0.019131733973821004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,1536,0.023409066597620647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,1024,0.016740266482035318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,1024,0.012981333335240684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,1024,0.0209279994169871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,1024,0.014674133062362671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,768,0.010739200313886007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,768,0.019962666432062785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,2560,0.02879253427187602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,512,0.008565333485603333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,512,0.01758506695429484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,256,0.00555733342965444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,256,0.016621866822242738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,256,0.008499200145403545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,128,0.0046528001626332605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,128,0.016033066312472026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,2048,0.026181334257125856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,128,0.007694933315118153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,64,0.004349866509437561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,32,0.004598399996757508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,7168,0.07519466876983642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,64,0.016247466206550598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,32,0.01694186727205912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,6144,0.06529173453648886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,4096,0.04475520054499309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,65536,0.31898454030354817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,65536,0.5007242520650228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,51200,0.40251839955647783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,65536,0.5509642918904623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,16384,0.1496448040008545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,16384,0.1020095984141032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,51200,0.2510325272878011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,12288,0.07367573579152426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,12288,0.10513493220011394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,51200,0.42914454142252606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,10240,0.08934826850891113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,10240,0.06466026703516642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,12288,0.11002453168233235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,10240,0.09530986944834391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,8192,0.0541375994682312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,8192,0.07918826738993326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,7168,0.059970132509867345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,7168,0.050388268629709874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,6144,0.055606400966644286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,6144,0.04526826540629069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,5120,0.046632532278696695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,1536,0.019705599546432494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,5120,0.040402134259541825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,4096,0.036551467577616376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,4096,0.03539093335469563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,768,0.013281066219011942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,3584,0.03572800159454346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,3584,0.033090132474899295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,512,0.010382933417956035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,8192,0.07194986343383789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,3072,0.030007465680440264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,3072,0.03070826729138692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,2560,0.025240532557169598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,2560,0.027888000011444092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,2560,0.027562665939331054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,2048,0.021066667636235555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,2048,0.025214932362238568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,16384,0.14427839914957682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,1536,0.019169066349665323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,1536,0.023078399896621703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,7168,0.06993280251820883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,1024,0.012486400206883748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,6144,0.061341865857442224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,1024,0.020520534118016562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,768,0.010577066739400228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,1024,0.013959466417630514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,768,0.019334399700164796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,768,0.012499200304349263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,5120,0.051982935269673666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,512,0.00817920019229253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,512,0.017258665959040322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,256,0.005571199953556061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,256,0.01665493349234263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,4096,0.04148373206456502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,256,0.007666133344173431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,128,0.004570666452248891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,128,0.01593066652615865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,128,0.007045333087444305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,64,0.004124800115823746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,64,0.016084266702334087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,32,0.01635199983914693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,3584,0.03679466644922892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,3072,0.03224426706631978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,65536,0.2583456039428711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,65536,0.41359039942423503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,2048,0.02292799949645996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,65536,0.358732795715332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,51200,0.3278869311014811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,51200,0.2092426617940267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,16384,0.10211626688639323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,12288,0.07959252993265788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,16384,0.07594559987386068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,32,0.004204800228277842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,12288,0.06176000038782755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,10240,0.06908799807230631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,10240,0.05410453478495279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,8192,0.05399466753005981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,8192,0.04513920148213704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,8192,0.0498250683148702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,7168,0.04809919993082683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,7168,0.04183893203735352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,1536,0.018269866704940796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,6144,0.04301226536432902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,512,0.009430399537086487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,6144,0.03859519958496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,5120,0.037330134709676104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,5120,0.03454826672871907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,4096,0.02905813256899516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,12288,0.07270932992299398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,4096,0.03127893408139547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,3584,0.025976532697677614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,4096,0.02754773298899333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,3584,0.028485333919525145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,3072,0.023143466313680014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,3072,0.026179200410842894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,51200,0.2845386823018392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,16384,0.09518506526947021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,2560,0.022362667322158813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,2560,0.024175999561945598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,2048,0.016557866334915163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,2048,0.022270933787027995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,10240,0.06203200022379557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,1536,0.012668800354003907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,1536,0.02078933318456014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,1536,0.0141184002161026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,1024,0.009461333354314167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,1024,0.019010132551193236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,1024,0.01162453293800354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,768,0.008184533317883809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,768,0.018003199497858682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,7168,0.044609065850575766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,512,0.00678719977537791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,512,0.016671999295552572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,512,0.008461866776148479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,256,0.004609066744645437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,256,0.015963733196258545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,6144,0.03910826841990153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,128,0.004108799993991852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,5120,0.03271573384602865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,128,0.015607466300328573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,64,0.0037813333173592886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,64,0.01585706671079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,32,0.004141866664091746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,32,0.01628373364607493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,3584,0.02485333283742269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,3072,0.023041067520777385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,65536,0.2445141315460205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,65536,0.3628650665283203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,51200,0.28850879669189455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,51200,0.18625814119974773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,256,0.0075541332364082335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,16384,0.10489813486735027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,16384,0.08011840184529623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,16384,0.09098133246103922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,51200,0.26302080154418944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,12288,0.10533226331075032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,12288,0.05687893231709799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,12288,0.06729066371917725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,2560,0.01952000061670939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,10240,0.06788693269093832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,10240,0.0511573314666748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,10240,0.056645333766937256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,8192,0.042471468448638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,8192,0.07312533060709635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,7168,0.04192426602045695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,7168,0.03961919943491618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,7168,0.042498131593068436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,2048,0.016706132888793947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,6144,0.037445334593455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,6144,0.03638399839401245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,5120,0.03193813363711039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,6144,0.03684053421020508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,5120,0.032808534304300946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,5120,0.03136533300081889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,4096,0.026220800479253133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,4096,0.028983465830485028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,3584,0.023746132850646973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,3584,0.027273599306742353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,3072,0.021104000012079873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,3072,0.025873066981633504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,768,0.010272000233332317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,2560,0.018173867464065553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,2560,0.02405440012613932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,128,0.006877866884072621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,2048,0.014670933286348978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,2048,0.022273067633310953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,65536,0.3410304069519043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,1536,0.012998400131861368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,1536,0.020524799823760986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,1536,0.01288746694723765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,1024,0.008981333176294962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,1024,0.01964799960454305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,8192,0.04826773405075073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,768,0.007749333480993907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,768,0.01732586622238159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,768,0.008851200342178345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,512,0.006223999957243601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,4096,0.02615999976793925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,3584,0.023779199520746867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,512,0.016992000738779704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,256,0.0047423998514811196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,128,0.004277333120505015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,256,0.01616106629371643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,3072,0.02090453306833903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,128,0.015784533818562825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,64,0.004049066702524821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,64,0.01593706707159678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,32,0.004196266829967499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,32,0.016156799594561257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,2560,0.018191999197006224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,65536,0.31486399968465173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,65536,0.21645973523457845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,65536,0.3035178820292155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,51200,0.24958826700846354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,2048,0.015577600399653117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,51200,0.17311360041300455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,16384,0.09347413380940756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,16384,0.0663424015045166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,12288,0.09131093025207519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,12288,0.05400533278783163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,10240,0.07291093667348227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,12288,0.06363733212153116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,128,0.006000000238418579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,10240,0.04964480002721151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,10240,0.053003732363382974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,8192,0.04140266577402751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,8192,0.04376426537831624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,8192,0.04422826766967773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,7168,0.03661973476409912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,7168,0.038894931475321456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,6144,0.032630399862925215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,7168,0.03794346650441487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,6144,0.035335465272267656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,5120,0.02912213404973348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,5120,0.03226240078608195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,4096,0.02318293253580729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,4096,0.028483200073242187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,4096,0.02515520056088765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,3584,0.020824533700942994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,3584,0.02616853316624959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,1024,0.010610133409500122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,3072,0.018819200992584228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,3072,0.02449173331260681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,3072,0.02033920089403788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,2560,0.015345066785812378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,512,0.007719466586907704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,2560,0.022785067558288574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,256,0.00654720018307368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,2560,0.017496534188588462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,2048,0.01274773379166921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,1536,0.010498133301734925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,2048,0.021734400590260824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,1536,0.01991573373476664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,1536,0.01281599998474121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,1024,0.008100266754627227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,1024,0.010237866640090942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,1024,0.017778132359186807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,768,0.006938666601975759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,768,0.01679253379503886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,768,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,512,0.005312000215053558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,512,0.016150400042533875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,512,0.007446399827798207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,256,0.0042346666256586705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,256,0.01579839984575907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,128,0.0038111999630928038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,128,0.015460266669591268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,51200,0.2422815958658854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,16384,0.08382186889648438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,64,0.00346666673819224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,64,0.015410133202870688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,32,0.0037962667644023894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,32,0.015574399630228677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,65536,0.2553130626678467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,6144,0.034392531712849936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,5120,0.02909333308537801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,65536,0.18792640368143718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,51200,0.20328960418701172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,3584,0.022273067633310953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,51200,0.14922666549682617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,16384,0.0739135980606079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,16384,0.059661865234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,12288,0.05677013397216797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,12288,0.04842986663182576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,2048,0.014759467045466105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,10240,0.0486847996711731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,10240,0.042733867963155106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,256,0.006442666550477346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,128,0.005897599955399831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,8192,0.03774826526641846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,8192,0.04157973527908325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,7168,0.034732798735300704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,7168,0.03482666810353597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,7168,0.03777066469192505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,6144,0.0303274671236674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,6144,0.03205759922663371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,5120,0.025869866212209065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,5120,0.028548266490300494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,5120,0.028226133187611895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,4096,0.021418666839599608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,8192,0.03914133310317993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,65536,0.28088960647583006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,3584,0.019871999820073448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,3584,0.02436586618423462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,51200,0.22518293062845865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,16384,0.07677333354949951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,3072,0.018572799364725747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,3072,0.02340373396873474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,2560,0.014827733238538107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,12288,0.06030826568603516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,2560,0.022325332959493002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,2048,0.012604799866676331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,2048,0.021066667636235555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,10240,0.05170773267745972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,4096,0.02564053336779277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,1536,0.010295466581980387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,1536,0.01932586630185445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,1536,0.011621333161989848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,1024,0.008027733365694682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,1024,0.017026132345199584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,1024,0.00939626693725586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,768,0.006363733112812043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,768,0.016825600465138753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,768,0.007928533355395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,512,0.005339733262856802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,512,0.016006400187810264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,512,0.00698773314555486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,256,0.004095999896526337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,256,0.01558080017566681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,128,0.0038474666575590765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,128,0.016089600324630738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,6144,0.0330677330493927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,64,0.003446399917205175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,64,0.015438933173815408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,32,0.0034826666116714476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,32,0.015441067020098367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,65536,0.2088640054066976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,65536,0.16275307337443035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,4096,0.023868799209594727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,3584,0.020641066630681357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,51200,0.166157865524292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,65536,0.2643167972564697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,51200,0.13111039797465007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,3072,0.019010132551193236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,16384,0.05872533321380615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,16384,0.05641280015309652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,2560,0.016748799880345663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,12288,0.04748266537984212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,2048,0.013977600137392679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,12288,0.05339200099309286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,10240,0.050273064772288004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,10240,0.03859093189239502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,8192,0.029256532589594524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,8192,0.032389332850774125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,7168,0.02677226662635803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,7168,0.029845333099365233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,256,0.006093866626421611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,6144,0.026296534140904743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,6144,0.02914773424466451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,128,0.005576533575852713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,12288,0.05051093498865763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,5120,0.022374399503072104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,5120,0.025592533747355144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,4096,0.01660160024960836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,4096,0.02352959911028544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,4096,0.021492266654968263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,3584,0.014773333072662353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,3584,0.02227413256963094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,3584,0.019638399283091225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,3072,0.014289066195487976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,16384,0.06714133421579996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,51200,0.20305493672688804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,3072,0.021498666206995646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,2560,0.01132480005423228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,3072,0.017461333672205606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,10240,0.04546133279800415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,2560,0.02087786595026652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,2048,0.009921066959698995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,2048,0.019377066691716512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,8192,0.036741332213083906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,1536,0.008318933347860973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,1536,0.01762239933013916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,1024,0.006428800026575724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,7168,0.03227199912071228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,1024,0.0164192001024882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,768,0.0054730668663978575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,768,0.0167797327041626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,6144,0.028495999177296956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,512,0.004654933512210846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,5120,0.024990934133529662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,512,0.015963733196258545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,256,0.00391146664818128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,256,0.015414399902025857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,2048,0.013086932897567748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,128,0.0034986667335033415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,128,0.015178666512171427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,128,0.005771733323733012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,64,0.0033098667860031127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,64,0.015343999862670899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,32,0.0035391998787721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,32,0.01548799971739451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,65536,0.18373440106709799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,65536,0.15220799446105956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,512,0.0070592001080513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,51200,0.14885652860005696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,51200,0.12048213481903076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,16384,0.05627520084381103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,16384,0.05453973213831583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,2560,0.015247999628384908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,12288,0.045507200558980304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,1536,0.011354666948318482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,1024,0.008825600147247314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,12288,0.044436268011728924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,10240,0.04690560102462769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,768,0.007950933277606964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,8192,0.03773866494496663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,8192,0.034995198249816895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,8192,0.03799893458684285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,7168,0.03351039886474609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,7168,0.031897600491841635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,256,0.006331733365853627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,6144,0.030495999256769817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,6144,0.03020053307215373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,10240,0.03961919943491618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,5120,0.01874879995981852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,5120,0.026764800151189167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,65536,0.25296640396118164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,4096,0.02147093415260315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,4096,0.024204800526301064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,51200,0.20186559359232584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,3584,0.01971199909845988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,3584,0.02262079914410909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,16384,0.07117119630177816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,3584,0.01926079988479614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,3072,0.015117866794268289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,3072,0.02166293263435364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,2560,0.012801067034403483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,12288,0.054117333889007566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,2560,0.0206058661142985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,2048,0.010936533411343891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,10240,0.04570026795069377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,2048,0.02003306746482849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,2048,0.012876799702644348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,1536,0.008730666836102803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,1536,0.017407999436060587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,1024,0.0069472000002861025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,1024,0.0168778657913208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,7168,0.03408960103988647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,768,0.0054058666030565895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,768,0.016290133198102318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,6144,0.029309866825739543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,512,0.004657066861788432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,512,0.016014933586120605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,512,0.006545066833496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,256,0.0038986665507157645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,5120,0.025496532519658405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,256,0.015399466951688132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,128,0.0035349334279696143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,128,0.014993066589037577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,4096,0.02158400019009908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,64,0.0033429334561030067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,64,0.015133866667747497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,32,0.003568000098069509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,32,0.015296000242233276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,3072,0.017732266585032144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,2560,0.015068800250689188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,65536,0.16309973398844402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,65536,0.13946879704793294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,1536,0.010293333729108175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,51200,0.133188263575236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,51200,0.11179520289103191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,16384,0.05303786595662435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,1024,0.008424533406893413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,16384,0.0467029333114624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,16384,0.0674570639928182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,12288,0.04059520165125529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,12288,0.041051733493804934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,12288,0.053174400329589845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,10240,0.03548479874928792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,10240,0.03463146686553955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,8192,0.025226666529973345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,768,0.007427200178305308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,8192,0.031048534313837688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,7168,0.028699733813603717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,7168,0.02688000003496806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,256,0.005809066692988077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,128,0.0053247998158137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,6144,0.021358933051427206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,6144,0.02905279994010925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,6144,0.02813653349876404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,5120,0.016511999567349753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,5120,0.025135999917984007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,5120,0.024664533138275147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,4096,0.01736533244450887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,4096,0.02390399972597758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,3584,0.014729600151379904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,4096,0.02050986687342326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,3584,0.022395733992258707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,3072,0.013924266894658408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,3072,0.02180373271306356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,51200,0.19404479662577312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,65536,0.23802453676859536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,2560,0.011480533083279927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,2560,0.02029013236363729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,2048,0.009481599926948548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,2048,0.012104533116022746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,2048,0.019105066855748497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,1536,0.007910400132338206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,1536,0.01809813380241394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,10240,0.04429440100987752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,1024,0.006227200229962667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,1024,0.0165173331896464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,768,0.00514986664056778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,768,0.015947733322779337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,8192,0.03636586666107178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,512,0.004323199888070424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,7168,0.03382186492284139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,512,0.016006400187810264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,256,0.0038090666135152185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,256,0.015520000457763672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,256,0.005526400109132131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,128,0.003525333354870478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,128,0.015140266219774882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,3584,0.018577067057291667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,512,0.006244266529877981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,64,0.0033439998825391137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,32,0.0035434665779272715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,64,0.01574399967988332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,32,0.01511679987112681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,3072,0.01730560064315796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,65536,0.14038292566935223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,65536,0.12586239973704022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,51200,0.11192320187886555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,65536,0.23096213340759278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,51200,0.10060266653696696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,16384,0.042599467436472575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,16384,0.04328533411026001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,12288,0.03291306694348653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,2560,0.014553599556287131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,12288,0.03577066659927368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,12288,0.05080106655756632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,10240,0.02513813376426697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,10240,0.043093331654866535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,10240,0.03266986608505249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,8192,0.023312000433603923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,8192,0.02794666687647502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,1536,0.009781333804130554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,7168,0.018784000476201376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,7168,0.026686932643254595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,1024,0.00804799993832906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,6144,0.018415999412536622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,768,0.007110400001207988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,6144,0.027704532941182452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,6144,0.025433599948883057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,5120,0.016390400131543477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,4096,0.013512532909711203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,5120,0.023128533363342287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,4096,0.020139733950297035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,4096,0.021593600511550903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,3584,0.01060479978720347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,3584,0.02091839909553528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,3584,0.017914666732152303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,3072,0.01083733340104421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,3072,0.016607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,2560,0.008640000224113464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,128,0.0051584000388781226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,3072,0.019937066237131755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,2048,0.008482133348782856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,2560,0.019203199942906698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,2048,0.017579734325408936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,51200,0.18732372919718426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,1536,0.006626133124033611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,1536,0.017072000106175742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,1536,0.009797333677609762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,16384,0.06586666504542033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,1024,0.00554666668176651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,1024,0.007930666704972585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,768,0.004358399907747904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,768,0.016167466839154564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,8192,0.0347434679667155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,7168,0.032264532645543416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,512,0.0038624001046021783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,512,0.015592533349990844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,256,0.003588266670703888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,256,0.015188266833623251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,5120,0.023853866259257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,128,0.003316266586383184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,128,0.014725333452224732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,2560,0.013899733622868856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,64,0.0030730667213598887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,64,0.014889599879582724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,32,0.0031968000034491217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,1024,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,32,0.015079466501871744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,65536,0.11495253245035808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,65536,0.11258347034454345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,51200,0.09051413536071777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,65536,0.21637439727783203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,51200,0.09019839763641357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,16384,0.03973226547241211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,16384,0.03970986604690552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,2048,0.011532800396283467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,12288,0.03095146616299947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,12288,0.03299626708030701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,10240,0.02755413254102071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,10240,0.028776532411575316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,10240,0.03877546787261963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,8192,0.019189333915710448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,8192,0.027974400917689008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,8192,0.031531733274459836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,768,0.007188266515731812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,7168,0.017366399367650352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,7168,0.026023467381795246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,6144,0.015099733074506124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,6144,0.02483733296394348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,512,0.006246399879455566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,5120,0.017343999942143758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,256,0.005570133527119955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,5120,0.024060799678166708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,5120,0.02201919953028361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,4096,0.013707733154296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,4096,0.02180160085360209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,3584,0.012641066312789917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,3584,0.02114026745160421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,128,0.0051701332132021586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,3072,0.011453866958618164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,3072,0.020090667406717937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,51200,0.16104532877604166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,16384,0.05944000085194906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,2560,0.00997226635615031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,2560,0.018733867009480796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,2048,0.008550399541854858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,2048,0.018242132663726807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,12288,0.046470399697621664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,1536,0.006274133423964183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,1536,0.01754666765530904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,7168,0.02844480077425639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,1024,0.004728533327579498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,1024,0.01663253307342529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,6144,0.02563733259836833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,768,0.004316799839337667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,768,0.01616213321685791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,768,0.00697386662165324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,4096,0.01821546753247579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,512,0.003866666555404663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,512,0.015660799543062844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,256,0.0035103999078273775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,3584,0.016862932840983072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,256,0.005639466643333435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,256,0.014968533317248026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,128,0.0032255999743938447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,3072,0.015064533551534018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,128,0.014829867084821067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,64,0.003047466774781545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,32,0.003235200047492981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,64,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,32,0.014849066734313965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,65536,0.0939573367436727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,65536,0.10229120254516602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,65536,0.21455680529276527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,51200,0.08094720045725504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,2560,0.012899200121561686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,2048,0.010648533701896667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,16384,0.02993173400561015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,16384,0.03728640079498291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,16384,0.06073493162790934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,12288,0.02413653333981832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,12288,0.03144320050875346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,10240,0.020870399475097657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,10240,0.02834026614824931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,51200,0.07381119728088378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,1536,0.0093941330909729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,8192,0.017375999689102174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,8192,0.02639893293380737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,1024,0.0075530668099721265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,7168,0.01649173299471537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,7168,0.02434773246447245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,512,0.006187733511130015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,6144,0.014792533715566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,6144,0.023128533363342287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,128,0.005157333115736643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,5120,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,5120,0.021781333287556968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,5120,0.0220853328704834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,4096,0.011001599828402202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,4096,0.01969919999440511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,51200,0.17235093116760253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,3584,0.010045866171518963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,3584,0.01889066696166992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,3584,0.01639893352985382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,3072,0.008459732929865519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,3072,0.01824959913889567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,12288,0.04620159864425659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,2560,0.008196266492207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,2560,0.0181386669476827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,10240,0.04011733531951904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,2048,0.007266133526961009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,2048,0.017163733641306557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,2048,0.010924800237019857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,1536,0.006029866635799408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,8192,0.03259413242340088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,1536,0.01664959987004598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,1024,0.0047872001926104225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,7168,0.029636265834172566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,1024,0.015941333770751954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,1024,0.007589333256085714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,768,0.004314666489760081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,768,0.015732266505559287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,512,0.0037664001186688742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,512,0.015969066818555196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,512,0.005898666878541311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,256,0.0033791999022165934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,6144,0.02651840051015218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,256,0.015603199601173401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,256,0.005434666574001312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,128,0.0031615999837716425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,128,0.005087999999523163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,128,0.014765866597493491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,64,0.002995199958483378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,32,0.0030069333811601003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,64,0.014903466900189719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,32,0.01482133368651072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,65536,0.064683731396993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,65536,0.09329386552174887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,51200,0.06231573422749838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,4096,0.01848213275273641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,51200,0.07308373451232911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,16384,0.023030400276184082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,16384,0.03157119949658711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,16384,0.0543999989827474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,12288,0.019078399737675986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,12288,0.028919466336568195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,10240,0.01530239979426066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,10240,0.026260266701380413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,10240,0.035867734750111895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,8192,0.013025066256523133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,8192,0.02380160093307495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,3072,0.01453013320763906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,7168,0.012075733145078022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,7168,0.022352000077565513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,7168,0.0266485333442688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,6144,0.012108799815177918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,6144,0.021654399236043294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,2560,0.012613333264986672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,5120,0.01076586643854777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,5120,0.02086506684621175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,5120,0.0197269340356191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,4096,0.009346133470535279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,4096,0.019170133272806804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,1536,0.009070932865142822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,4096,0.01665600041548411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,3584,0.008668800195058186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,3584,0.018427733580271402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,3072,0.007805866499741872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,3072,0.018616533279418944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,768,0.006734933455785115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,2560,0.006970666845639547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,2560,0.017883733908335368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,2560,0.012054399649302164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,2048,0.006227200229962667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,2048,0.01763413349787394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,65536,0.18793706893920897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,1536,0.00520000010728836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,1536,0.016775466998418174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,51200,0.147816530863444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,12288,0.04242773453394572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,1024,0.004473599791526795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,1024,0.007340799768765767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,1024,0.015787733594576518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,768,0.004054400076468786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,768,0.015620266397794088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,768,0.006495999793211619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,512,0.0036864000062147772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,512,0.015155200163523355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,512,0.0058890665570894875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,256,0.003302400062481562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,256,0.01463573376337687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,256,0.0052714665730794275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,128,0.0030752000709374744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,128,0.014760532975196838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,6144,0.02320746580759684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,64,0.0028661333024501802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,64,0.01464959979057312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,32,0.0029056000212828318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,32,0.014754133423169455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,65536,0.05135039885838827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,65536,0.07985066572825114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,3584,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,8192,0.030271999041239422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,65536,0.162337064743042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,3072,0.01376106639703115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,51200,0.042266666889190674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,16384,0.016300800442695617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,51200,0.06403199831644693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,51200,0.12947839895884197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,12288,0.015289599696795145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,16384,0.029951999584833782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,16384,0.04444053173065186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,10240,0.011970133582750956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,10240,0.02615999976793925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,8192,0.012607999642690024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,8192,0.023510400454203287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,1536,0.008754133184750875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,7168,0.013683199882507324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,7168,0.0223306675752004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,12288,0.02839039961496989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,6144,0.014252799749374389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,6144,0.02140799959500631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,2048,0.01018346647421519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,5120,0.010761599739392598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,128,0.005041066805521647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,5120,0.021172267198562623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,4096,0.009236266215642292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,4096,0.01931519905726115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,3584,0.008477866649627686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,3584,0.018630399306615194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,12288,0.03512746493021647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,3072,0.007787733276685078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,3072,0.017918932437896728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,10240,0.029921066761016846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,2560,0.006614399949709575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,3072,0.011554132898648579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,2560,0.01728640000025431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,8192,0.024719999233881632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,2048,0.00588266650835673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,2048,0.0169322669506073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,1536,0.005222400029500326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,1536,0.01667840083440145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,7168,0.022501333554585775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,1024,0.0045503998796145115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,1024,0.01567359964052836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,6144,0.01969919999440511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,768,0.004112000018358231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,768,0.01552959978580475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,768,0.006018133461475372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,4096,0.014327466487884521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,5120,0.016499200463294984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,512,0.0036831999818483984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,256,0.003305600086847941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,512,0.005669333537419637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,512,0.015281066298484802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,256,0.014801067113876343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,128,0.002962133288383484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,128,0.004949333270390829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,128,0.014692266782124838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,64,0.002902399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,64,0.014390400052070618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,32,0.0028053333361943563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,32,0.014634666840235391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,65536,0.039527467886606854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,65536,0.07349119981129965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,65536,0.12553919951121012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,51200,0.03305173317591349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,51200,0.05931413173675537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,16384,0.013300266861915589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,16384,0.027688533067703247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,16384,0.0337066650390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,12288,0.010317867000897724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,12288,0.023982934157053628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,12288,0.026594134171803792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,10240,0.0115146666765213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,10240,0.022358399629592896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,2560,0.010328533252080281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,8192,0.009550933043162029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,3584,0.013215999801953635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,2048,0.009034666419029235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,8192,0.018179200092951455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,7168,0.022510933876037597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,1536,0.008060800035794576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,7168,0.016643200318018594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,6144,0.008887466788291932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,6144,0.021221333742141725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,1024,0.006741333504517872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,5120,0.008288000027338665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,5120,0.020231467485427857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,5120,0.012676266829172769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,4096,0.007294933497905731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,4096,0.01881386637687683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,8192,0.020897066593170165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,3584,0.00768746683994929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,7168,0.009733333190282186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,3584,0.018267732858657838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,3072,0.00699946681658427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,3072,0.017849600315093993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,256,0.005155199766159057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,2560,0.006257066627343495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,2560,0.017467733224232992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,51200,0.09687999884287515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,2048,0.00553599993387858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,2048,0.016775466998418174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,10240,0.022065067291259767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,1536,0.004890666902065277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,1536,0.016229333480199178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,6144,0.014678399761517844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,1024,0.004192000130812327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,1024,0.015876266360282897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,768,0.0038751999537150065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,768,0.015491200486818948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,4096,0.011291733384132386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,512,0.0035349334279696143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,512,0.015222400426864624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,3584,0.010733866691589355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,256,0.0031583999594052637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,3072,0.009497599800427754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,256,0.014991999665896098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,256,0.004986666639645894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,128,0.002917333443959554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,128,0.014691199858983359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,128,0.004852266609668731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,64,0.00276053324341774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,32,0.0027797333896160126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,64,0.014604799946149192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,32,0.014711466431617738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,65536,0.02691199978192647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,2560,0.008691199620564779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,51200,0.02258453369140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,65536,0.10198506514231365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,65536,0.0667680025100708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,16384,0.010095999638239542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,51200,0.05276906490325928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,16384,0.027411200602849323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,16384,0.025676800807317095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,12288,0.008936533331871032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,12288,0.022825600703557332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,12288,0.021415466070175172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,10240,0.01016213297843933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,2048,0.0075914666056633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,8192,0.00897920032342275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,10240,0.018658133347829182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,10240,0.021373866001764934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,7168,0.008008533219496409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,8192,0.015526400009791056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,8192,0.02107306718826294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,7168,0.020113066832224528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,6144,0.0076330666740735366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,6144,0.012938666343688964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,6144,0.01956693331400553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,5120,0.007012266914049785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,5120,0.01962560017903646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,5120,0.011461333433787028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,1536,0.007285333176453908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,4096,0.0187882661819458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,3584,0.006712533533573151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,3584,0.01858773430188497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,1024,0.00625600020090739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,3072,0.006357333560784657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,3072,0.017890133460362754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,3072,0.008905599514643352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,768,0.005660800139109293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,2560,0.0061930666367212926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,2560,0.008188800017038981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,2048,0.005577600002288819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,2048,0.016758400201797485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,2048,0.007316266496976216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,1536,0.004830933113892873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,1536,0.016354133685429893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,4096,0.0064416001240412395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,1024,0.004105599969625473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,4096,0.010532266894976298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,1024,0.00592853327592214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,1024,0.015460266669591268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,768,0.00384853333234787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,512,0.005356800059477488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,768,0.015343999862670899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,512,0.0034293333689371743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,512,0.014946132898330688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,256,0.003134933362404505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,256,0.01455466647942861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,51200,0.08066346645355224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,2560,0.017773866653442383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,128,0.0029674666623274487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,128,0.014476799964904785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,7168,0.014217600226402283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,64,0.00277866671482722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,64,0.014814933141072592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,32,0.002794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,32,0.014562132954597472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,65536,0.02007253368695577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,65536,0.06181973218917847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,3584,0.009976533055305482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,65536,0.1005237340927124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,51200,0.0151829332113266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,51200,0.04695680141448975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,51200,0.07947200139363607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,16384,0.007996800045172375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,16384,0.02360853354136149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,12288,0.0075093333919843035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,12288,0.0219541331132253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,12288,0.021673599878946938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,10240,0.007041066884994507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,10240,0.02070186734199524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,10240,0.018770132462183634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,8192,0.0066655998428662615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,8192,0.019825067122777304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,8192,0.015544533729553223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,7168,0.006443733473618825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,1536,0.006902400155862172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,7168,0.019793067375818887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,6144,0.006159999966621399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,7168,0.014247467120488485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,768,0.005681066711743673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,6144,0.012910933295885722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,5120,0.006604800124963124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,512,0.005253333350022634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,6144,0.01956160068511963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,4096,0.006213333209355672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,5120,0.011434666315714518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,5120,0.019708800315856933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,3584,0.00659093310435613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,4096,0.010128000378608703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,3584,0.018067200978597008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,3072,0.00625493327776591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,3584,0.009896533687909444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,2560,0.006153599917888641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,3072,0.017691733439763387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,2560,0.017154133319854735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,256,0.004965333143870035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,2048,0.005437866846720377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,2048,0.017078399658203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,1536,0.004661333560943603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,1536,0.016327466567357382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,1536,0.006886399785677592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,1024,0.004078933348258337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,1024,0.015565866231918335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,1024,0.006078933179378509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,768,0.0036938667297363283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,128,0.004709333181381226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,768,0.015271466970443726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,512,0.0034005333979924522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,768,0.005490133166313171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,256,0.0030400000512599947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,512,0.015228799978892007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,256,0.004953599969546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,256,0.014778666694959006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,128,0.0029045333464940387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,128,0.004746666550636292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,128,0.014667733510335287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,64,0.002733866622050603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,32,0.002713600049416224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,64,0.014498133460680643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,4096,0.018849066893259683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,32,0.01460693379243215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,65536,0.010920533537864685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,51200,0.010135466853777569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,16384,0.007584000130494435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,65536,0.05740799903869629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,51200,0.04244266748428345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,12288,0.006551466882228851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,16384,0.02339413364728292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,10240,0.00647680014371872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,12288,0.021678932507832847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,8192,0.006326400240262349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,10240,0.02097599903742472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,7168,0.006169599791367849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,6144,0.006018133461475372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,8192,0.02007360061009725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,7168,0.021040000518163047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,5120,0.006438399851322174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,6144,0.019668267170588175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,16384,0.02725013295809428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,4096,0.0060576001803080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,5120,0.019589332739512126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,3584,0.006493866443634033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,4096,0.018902399142583213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,3584,0.018228266636530557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,2560,0.006025599936644236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,3072,0.017782400051752724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,2048,0.005410133302211762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,2560,0.01721280018488566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,1536,0.004734933376312256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,2048,0.016708266735076905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,1536,0.016427733500798545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,1024,0.003988266736268997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,768,0.0037162666519482933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,1024,0.01564586659272512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,512,0.003390933324893316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,768,0.015282133221626281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,256,0.003028266628583272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,512,0.015266133348147073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,128,0.002812800059715907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,256,0.014786133170127868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,64,0.0027082666754722597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,128,0.014528000354766845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,64,0.014486400286356607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,32,0.014318933089574179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,65536,0.00992746651172638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,65536,0.0564575990041097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,51200,0.008733866612116496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,51200,0.04121599992116292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,16384,0.006389333307743073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,3072,0.008711466193199157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,12288,0.006179200112819671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,12288,0.021309866507848104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,2560,0.008075733482837678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,10240,0.006489600241184235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,8192,0.0063178668419520065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,2048,0.0072170664866765336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,7168,0.006146133442719777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,7168,0.020125865936279297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,6144,0.005970133344332377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,6144,0.02050559918085734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,5120,0.006469333171844482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,5120,0.019870932896931967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,4096,0.005974400043487549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,32,0.002616533388694127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,4096,0.01844693422317505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,3584,0.006404266754786174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,3584,0.018293333053588868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,3072,0.01835413376490275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,2560,0.006035199761390686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,512,0.005125333368778229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,2048,0.0053727999329566956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,2048,0.016800000270207723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,1536,0.004715733230113983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,1536,0.016345600287119545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,1024,0.004025600105524063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,16384,0.023840000232060753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,1024,0.015683199961980185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,768,0.003636266787846883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,512,0.0033791999022165934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,768,0.015396266182263692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,3072,0.006161066889762879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,256,0.003068800022204717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,512,0.014989866813023885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,128,0.002855466554562251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,10240,0.0209279994169871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,64,0.0026346666117509207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,128,0.014475733041763306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,32,0.002625066787004471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,64,0.014570666352907815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,32,0.014452266693115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,3072,0.006056533257166544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,2560,0.017245866854985557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,256,0.014763733744621277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,16384,0.4092384020487468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,8192,0.02029973268508911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,16384,0.7609642664591472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,12288,0.3167647997538249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,10240,0.4895914713541667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,12288,0.5802047729492188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,51200,1.2200448354085287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,10240,0.2679295857747396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,8192,0.38880319595336915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,8192,0.2511648019154867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,51200,2.381700388590495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,8192,0.4114879926045735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,7168,0.3412053426106771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,51200,2.3213737487792967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,7168,0.19545493125915528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,6144,0.3000629425048828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,6144,0.18248213132222493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,5120,0.24963626861572266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,5120,0.14783679644266765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,4096,0.2032661279042562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,5120,0.25167786280314125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,4096,0.12367146809895832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,3584,0.17957226435343426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,4096,0.19717760086059571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,3584,0.11098346710205079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,3072,0.1568661371866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,3072,0.09836373329162598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,3584,0.17436052958170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,2560,0.13268266518910726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,3072,0.1483733336130778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,2560,0.08569493293762206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,16384,0.7139328002929688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,2048,0.10897493362426758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,2048,0.07361706892649332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,2560,0.12575893402099608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,1536,0.08468906879425049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,1536,0.06128319899241129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,2048,0.09832213719685873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,1024,0.06035840113957723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,1024,0.04915200074513753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,768,0.04810026486714681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,768,0.042370132605234784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,768,0.045067731539408365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,512,0.03604480028152466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,512,0.035871998469034834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,512,0.034701867898305254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,256,0.022627200682957968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,10240,0.46870826085408523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,256,0.028514132897059126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,128,0.015019733707110086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,256,0.030270934104919434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,128,0.027379200855890913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,64,0.012075733145078022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,64,0.02617599964141846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,65536,32,0.01362879971663157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,32,0.028617600599924724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,6144,0.3114069302876791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,7168,0.36405760447184243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,65536,128,0.02569173375765483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,1536,0.07650880018870035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,1024,0.056321068604787194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,65536,1.3740416208902995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,16384,0.6630122502644856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,65536,2.6496841430664064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,16384,0.41222613652547196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,51200,2.0874399820963543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,51200,1.0788853963216147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,65536,12288,0.5398186365763347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,10240,0.24873919486999513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,12288,0.29517974853515627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,51200,1.8627253214518231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,10240,0.425930658976237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,12288,0.5889024098714193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,8192,0.33831679026285805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,10240,0.3993098576863607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,8192,0.2192608038584391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,7168,0.33530025482177733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,7168,0.1816736062367757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,6144,0.2769685427347819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,5120,0.234334929784139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,5120,0.13519466718037923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,5120,0.2142613410949707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,8192,0.3460895856221517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,4096,0.18586559295654298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,6144,0.16027092933654785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,4096,0.11312959988911946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,4096,0.16697386105855305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,3584,0.10163413683573405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,3584,0.166702938079834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,16384,0.5949759801228841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,65536,2.2998730977376303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,3072,0.09116586844126383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,3072,0.14153067270914715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,2560,0.07818880081176757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,2560,0.10621120134989422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,2560,0.13090879917144777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,2048,0.06638400157292684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,2048,0.09840640226999918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,1536,0.07421653270721436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,1536,0.055409065882364904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,12288,0.46120961507161456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,2048,0.0880949338277181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,1024,0.05326506694157919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,768,0.04191360076268514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,1536,0.0630677342414856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,1024,0.0539626677831014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,768,0.05443840026855469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,512,0.03400533199310303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,512,0.04300479888916016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,512,0.026817067464192705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,256,0.030734932422637938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,256,0.029793065786361695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,128,0.017574399709701538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,7168,0.30408105850219724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,128,0.026012800137201947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,128,0.01699413259824117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,6144,0.2599093278249105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,64,0.02391466697057088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,32,0.01176106631755829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,51200,32,0.0249781330426534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,65536,0.41949119567871096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,65536,0.7441439946492513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,51200,64,0.013223466277122498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,3584,0.14418880144755047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,3072,0.12419733206431072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,51200,0.5970357259114583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,51200,0.33290987014770507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,65536,0.7285354614257813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,16384,0.19242773056030274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,16384,0.13840853373209636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,12288,0.095524263381958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,12288,0.14734187126159667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,1024,0.049371735254923506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,10240,0.12462186813354492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,768,0.03623253504435221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,10240,0.08276800314585367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,12288,0.14559253056844074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,10240,0.12540053526560466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,8192,0.10000533262888592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,8192,0.0697546641031901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,7168,0.08788053194681802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,7168,0.06346880197525025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,7168,0.09461546738942464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,6144,0.07745066483815512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,51200,256,0.020108799139658608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,6144,0.057801600297292074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,6144,0.08161173661549886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,5120,0.050836265087127686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,4096,0.053882666428883876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,5120,0.06705066363016764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,4096,0.04437013467152913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,3584,0.047960531711578366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,4096,0.054674132664998376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,3584,0.04078293244043986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,3072,0.04227946599324544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,3584,0.04977493286132813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,3072,0.03784213463465373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,2560,0.0365503986676534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,3072,0.04186559915542602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,2560,0.03412479956944783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,2048,0.030264532566070555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,2048,0.03022293249766032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,2048,0.029407999912897748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,1536,0.024482132991154988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,1536,0.025942399104436235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,1024,0.017139200369517008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,51200,0.5729098637898763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,1024,0.02257066567738851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,768,0.013354667027791343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,1024,0.018309332927068076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,768,0.02091946601867676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,512,0.010040533542633057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,768,0.015785599748293556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,512,0.019700266917546592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,256,0.007238399982452392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,256,0.017178666591644288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,5120,0.06545279820760092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,128,0.005227733155091604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,128,0.016557866334915163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,128,0.00962666670481364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,64,0.004671999812126159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,64,0.016921599706014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,16384,32,0.0051466668645540874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,16384,32,0.017366399367650352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,1536,0.023052799701690673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,16384,0.19025920232137045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,65536,0.3535509427388509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,65536,0.6005472183227539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,8192,0.10756159623463948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,51200,0.47447894414265945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,51200,0.2665770689646403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,16384,0.14511040051778157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,12288,0.11203093528747558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,16384,0.11037973562876384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,51200,0.4853503863016765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,12288,0.07746133009592691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,12288,0.12640213171641032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,10240,0.06715733210245768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,8192,0.07605760097503662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,8192,0.0575274666150411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,512,0.01353493332862854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,8192,0.09578239917755127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,256,0.010663466652234395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,7168,0.06740266482035319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,7168,0.08329813480377198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,6144,0.059512531757354735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,6144,0.04787626663843791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,6144,0.07022186915079752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,5120,0.05006719827651977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,5120,0.06052373250325521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,5120,0.04292800029118855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,4096,0.04126826524734497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,4096,0.037774932384490964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,3584,0.03699306646982829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,4096,0.0480128010114034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,3584,0.034783999125162765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,10240,0.09477012952168783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,3072,0.032979200283686325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,3072,0.03432106574376424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,2560,0.027961599826812743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,2560,0.028943999608357744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,7168,0.05353066523869833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,2048,0.02346773346265157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,2048,0.025858134031295776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,65536,0.6294986724853515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,16384,0.16536213556925455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,1536,0.02081600030263265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,1536,0.020850133895874024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,1536,0.0234389324982961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,1024,0.012938666343688964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,1024,0.01614720026652018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,1024,0.02108586629231771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,768,0.010618666807810467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,768,0.01995733380317688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,768,0.013979732990264893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,10240,0.10851840178171794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,512,0.008526933193206788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,512,0.011485866705576579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,512,0.01773973306020101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,256,0.005523199836413065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,256,0.008724266290664673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,256,0.016590933005015053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,128,0.004427733520666758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,128,0.016479999820391337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,64,0.0041461333632469176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,64,0.01646293302377065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,12288,32,0.004404266675313314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,12288,32,0.016760534048080443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,16384,2560,0.037229867776234944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,3584,0.04350506862004598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,65536,0.489032522837321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,3072,0.036873598893483475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,2560,0.031880533695220946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,65536,0.30299094518025715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,51200,0.2441429297129313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,51200,0.39280640284220375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,16384,0.13563520113627117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,16384,0.08972053527832032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,2048,0.027309866746266682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,12288,0.1032960017522176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,12288,0.07164159615834555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,10240,0.08201599915822347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,51200,0.45770985285441085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,12288,0.11787306467692057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,10240,0.06340693235397339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,8192,0.0530784010887146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,10240,0.10486400127410889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,8192,0.07028266588846842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,8192,0.08903040091196696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,7168,0.062414932250976565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,7168,0.04944106737772624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,6144,0.0521226684252421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,6144,0.04452693462371826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,6144,0.06802240212758383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,5120,0.04434026479721069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,5120,0.039554135004679365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,4096,0.03591146469116211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,4096,0.03481706778208415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,3584,0.03368959824244182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,3584,0.03257386684417725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,3072,0.029459200302759808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,12288,128,0.007825066645940144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,3072,0.035563735167185466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,3072,0.030163200696309407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,2560,0.02518293261528015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,2560,0.027244800329208375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,2560,0.03100373347600301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,2048,0.02104426622390747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,2048,0.02492693265279134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,1536,0.016990933815638223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,1536,0.022873600323994957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,65536,0.5813845316569011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,16384,0.15412267049153644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,1024,0.012549333771069846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,1024,0.020503467321395873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,768,0.00960106650988261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,768,0.01967573364575704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,768,0.012899200121561686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,7168,0.07950186729431152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,512,0.008406399687131246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,512,0.018071466684341432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,256,0.005549866457780202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,256,0.016807466745376587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,256,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,5120,0.05746453205744425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,128,0.004615466793378194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,128,0.01680533289909363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,3584,0.041587201754252116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,64,0.004155733436346054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,64,0.015960533420244852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,10240,32,0.00425600012143453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,10240,32,0.016004266341527303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,65536,0.24670400619506835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,65536,0.3915306727091471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,2048,0.025613866249720257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,1536,0.01975040038426717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,51200,0.30900481541951497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,51200,0.20009919802347817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,65536,0.38026345570882164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,4096,0.04604373375574748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,16384,0.1140554666519165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,12288,0.07601706981658936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,16384,0.07289066314697265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,10240,0.06390080054601034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,12288,0.05899626811345419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,10240,0.05132373174031576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,8192,0.051677866776784265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,8192,0.04501440127690633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,1024,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,7168,0.045813333988189694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,7168,0.04102933406829834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,7168,0.04865386486053467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,512,0.010072533289591472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,6144,0.040644268194834396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,6144,0.0373525341351827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,5120,0.03465173244476318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,12288,0.0771882692972819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,10240,128,0.007091199855009715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,5120,0.03675093253453572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,5120,0.033763198057810466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,4096,0.028355199098587035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,4096,0.02965013384819031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,3584,0.025718400875727337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,3584,0.027789866924285887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,3584,0.0265664001305898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,3072,0.02302293380101522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,3072,0.025677865743637084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,3072,0.023821866512298583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,2560,0.019745065768559774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,2560,0.023834667603174844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,2560,0.02026346723238627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,2048,0.016127999623616537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,2048,0.022453333934148154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,51200,0.29525973002115885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,16384,0.1042517344156901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,1536,0.013933866222699483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,1536,0.014984533190727234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,1536,0.020957867304484047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,1024,0.009443199634552002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,1024,0.01914773384730021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,768,0.008113066852092742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,768,0.01837013363838196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,768,0.010702932874361675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,512,0.006694399813810985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,8192,0.054764799276987707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,512,0.016377600034077962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,256,0.004846933484077454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,256,0.016012799739837647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,256,0.007606400052706401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,128,0.004187733431657155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,128,0.015708800156911215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,6144,0.04280213514963786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,64,0.0037248000502586366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,64,0.01588586668173472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,8192,32,0.004148266712824504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,8192,32,0.016328533490498863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,10240,0.06604799826939901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,4096,0.029815467198689778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,65536,0.3729322751363119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,65536,0.22188800175984702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,51200,0.2989664077758789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,65536,0.34400320053100586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,16384,0.09764053026835123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,16384,0.06705386638641357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,51200,0.1789365291595459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,12288,0.06865599950154623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,2048,0.017670400937398276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,16384,0.09660053253173828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,12288,0.0548309326171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,12288,0.07010026772816977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,10240,0.057816533247629795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,10240,0.04863359928131104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,10240,0.06025173266728719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,8192,0.04644266764322917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,8192,0.04157973527908325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,8192,0.05163093407948812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,7168,0.041665065288543704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,6144,0.03705706596374512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,7168,0.044819200038909913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,6144,0.03571413358052571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,5120,0.03158079981803894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,6144,0.03954986731211345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,5120,0.03216000000635783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,4096,0.02574933369954427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,5120,0.033515731493632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,4096,0.027905066808064777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,3584,0.023681066433588662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,4096,0.02772586743036906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,3584,0.02622720003128052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,1024,0.01192639966805776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,3072,0.020840533574422202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,3584,0.024972800413767496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,2560,0.017923200130462648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,3072,0.022438400983810426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,2560,0.02330133318901062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,2048,0.014482133587201438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,2560,0.018990933895111084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,1536,0.011555199821790058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,2048,0.016555733482042947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,1536,0.020090667406717937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,1024,0.008872532844543457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,512,0.0086709330479304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,1024,0.019495467344919838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,1024,0.010980266332626342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,768,0.00796693315108617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,768,0.017391999562581383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,512,0.0060127998391787205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,51200,0.2841269175211588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,512,0.017214934031168617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,256,0.004463999966780345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,8192,128,0.006840533514817555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,512,0.008052266637484233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,256,0.016170666615168253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,128,0.004189866781234741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,128,0.0058783998092015585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,128,0.015577600399653117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,7168,0.038737066586812335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,64,0.0037248000502586366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,7168,32,0.0038719999293486277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,64,0.015687466661135355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,32,0.01583999991416931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,3072,0.024855466683705647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,7168,2048,0.021546665827433267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,65536,0.19820586840311688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,65536,0.3108394622802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,1536,0.013507200280825296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,16384,0.084989865620931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,51200,0.1603125254313151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,51200,0.2453930695851644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,16384,0.06206399997075399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,51200,0.25282559394836424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,16384,0.09026880264282226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,12288,0.050860798358917235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,12288,0.07348586718241373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,12288,0.06554773251215616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,10240,0.0448202649752299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,10240,0.059114666779836025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,8192,0.041137067476908366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,10240,0.05704533259073893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,8192,0.038754133383433025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,8192,0.04856640100479126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,7168,0.04416960080464681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,7168,0.036880000432332354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,6144,0.033418667316436765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,5120,0.03379520177841187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,5120,0.03028693397839864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,768,0.009194667140642803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,4096,0.028459733724594115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,4096,0.026154667139053345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,6144,0.039946667353312176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,4096,0.026401066780090333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,3584,0.025326933463414508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,3584,0.02504853407541911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,7168,256,0.0065290664633115125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,3072,0.022258132696151733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,3072,0.020652800798416138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,3072,0.024332799514134727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,2560,0.019326933224995933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,2560,0.022737065951029457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,2048,0.02138026754061381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,2048,0.015044266978899637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,1536,0.01197653313477834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,1536,0.019617066780726115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,65536,0.31811094284057617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,1024,0.009276800354321798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,1024,0.017630932728449504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,7168,0.04303786754608154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,768,0.007708799839019775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,6144,0.03694719870885213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,768,0.009092266360918682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,768,0.017298134167989095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,512,0.0057770664493242896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,512,0.01648640036582947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,512,0.007799466451009114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,256,0.004248533149560293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,5120,0.03219839930534363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,256,0.016263467073440552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,128,0.004249600072701773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,128,0.015564800302187601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,128,0.006045866509278615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,64,0.003684266656637192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,64,0.015691733360290526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,6144,32,0.003740799923737844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,6144,32,0.015804800391197204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,65536,0.17545493443806964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,65536,0.24989333152770996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,51200,0.19904640515645344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,51200,0.14146666526794432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,3584,0.024482132991154988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,2560,0.018408532937367758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,16384,0.07495466868082681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,2048,0.015711999932924905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,16384,0.05730346838633219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,51200,0.24083414077758789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,12288,0.0512117346127828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,12288,0.04667946497599284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,16384,0.08666880130767822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,10240,0.04746239980061849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,10240,0.04188266595204671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,1536,0.012891733646392822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,10240,0.053922132651011145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,12288,0.0631168007850647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,8192,0.03909333149592082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,7168,0.03135679960250855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,8192,0.03647146622339885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,7168,0.04072213172912598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,7168,0.03409493366877238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,6144,0.02790186603864034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,8192,0.04535253445307414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,6144,0.031174399455388385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,5120,0.02618880073229472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,6144,0.03688640197118123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,4096,0.021676800648371377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,5120,0.028062933683395387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,3584,0.019774933656056724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,4096,0.02563520073890686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,3584,0.02413653333981832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,3072,0.017087999979654947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,3072,0.023162666956583658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,3072,0.019825067122777304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,2560,0.014659200112024942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,2560,0.021523199478785195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,1024,0.010180266698201497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,2048,0.01111893355846405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,2048,0.020653865734736123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,6144,256,0.006563200056552887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,1536,0.01025493343671163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,1536,0.01945706605911255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,65536,0.29999361038208006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,1024,0.007970133423805236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,1024,0.016909867525100708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,1024,0.009700266520182292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,768,0.006780800223350525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,768,0.016459733247756958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,4096,0.02512213389078776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,5120,0.030179200569788615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,512,0.00543146679798762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,3584,0.022648533185323082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,512,0.007483733197053273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,512,0.016341333587964378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,256,0.004155733436346054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,128,0.003714133302370707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,256,0.015517866611480713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,128,0.005580799778302511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,128,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,64,0.003382399926582972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,5120,32,0.0035413332283496858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,64,0.01528320014476776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,5120,32,0.015532799561818442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,65536,0.15095787048339843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,65536,0.2043072064717611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,2560,0.01800853411356608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,2048,0.014647466937700906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,65536,0.2728597323099772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,16384,0.05784746805826822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,51200,0.1628885269165039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,51200,0.12368853092193603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,51200,0.2132703940073649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,12288,0.04668586651484172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,16384,0.05195519924163818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,16384,0.07508373260498047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,10240,0.04216213226318359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,12288,0.04350506862004598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,10240,0.03850666681925456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,8192,0.03322666684786479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,8192,0.034392531712849936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,1536,0.01218986709912618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,7168,0.031405866146087646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,6144,0.025963733593622845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,6144,0.02908693353335063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,256,0.006107733150323232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,6144,0.03207146724065145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,5120,0.023009065786997476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,5120,0.026360533634821576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,4096,0.01879040002822876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,4096,0.024184532960255942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,5120,768,0.008627200126647949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,7168,0.029021867116292316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,3584,0.017280000448226928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,3584,0.02100800077120463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,3584,0.02284053365389506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,3072,0.015094400445620219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,3072,0.021886932849884033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,2560,0.012853333353996277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,2560,0.020377600193023683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,10240,0.046404266357421876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,2048,0.010656000177065531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,12288,0.05680213371912638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,8192,0.04051946798960368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,2048,0.019629865884780884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,1536,0.009008000294367473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,1536,0.01752426624298096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,7168,0.03684266805648804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,1024,0.0069248000780741375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,1536,0.011502933502197266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,768,0.005684266487757364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,1024,0.016993065675099693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,768,0.016682666540145875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,768,0.008627200126647949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,512,0.004799999793370565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,512,0.01619733373324076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,256,0.0038293334345022834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,5120,0.02678080002466838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,256,0.015466666221618653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,4096,0.02249386707941691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,128,0.003409066547950109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,256,0.006451199948787689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,128,0.0058005332946777345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,128,0.01527466674645742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,32,0.0035573333501815797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,64,0.015340800086657206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,4096,32,0.015464533368746439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,65536,0.19305493036905924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,65536,0.13955626487731934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,51200,0.14376746813456218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,65536,0.2672053337097168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,3072,0.0180565337340037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,51200,0.11399359703063965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,2560,0.01637333333492279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,16384,0.0546175996462504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,16384,0.04599253336588542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,16384,0.07854506969451905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,12288,0.03757760127385457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,12288,0.03951359987258911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,4096,64,0.0032330666979153953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,10240,0.0315775990486145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,10240,0.03537919918696086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,10240,0.04952213366826375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,8192,0.025704532861709595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,8192,0.030293333530426025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,7168,0.02516266703605652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,7168,0.027884799242019653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,2048,0.01388159990310669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,1024,0.009144533673922222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,6144,0.022410666942596434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,6144,0.026205867528915405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,5120,0.01909546653429667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,4096,512,0.007480533421039581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,5120,0.024387200673421226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,4096,0.015428266922632852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,5120,0.02802666624387105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,4096,0.022530132532119752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,3584,0.013986133535703025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,3584,0.021668267250061036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,51200,0.21365973154703774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,3072,0.012523733576138816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,3072,0.02099840044975281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,2560,0.010923733313878376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,12288,0.05738986730575561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,2560,0.020204800367355346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,2560,0.016338133811950685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,2048,0.0093941330909729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,2048,0.01904426614443461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,1536,0.008011733492215473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,1536,0.017369600137074788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,1536,0.010947199662526448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,1024,0.006152533491452535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,1024,0.016707199811935424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,8192,0.041099735101064044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,768,0.005387733379999796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,7168,0.03755626678466797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,768,0.01646293302377065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,512,0.004609066744645437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,512,0.015960533420244852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,512,0.0069578667481740315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,256,0.0038592000802357995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,256,0.015613866845766702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,6144,0.033225599924723306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,128,0.003390933324893316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,128,0.015016532937685647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,4096,0.023695999383926393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,64,0.0031925333042939507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,64,0.015217066804567973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,3584,0.020746666193008422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3584,32,0.0033151999115943907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3584,32,0.015067733327547708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,65536,0.15892586708068848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,65536,0.12867093086242676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,51200,0.12924373149871826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,65536,0.2645269393920898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,51200,0.10540266831715901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,16384,0.04987733364105225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,16384,0.04340266784032186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,3072,0.01804800033569336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,12288,0.038114134470621744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,12288,0.03717333475748698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,2048,0.013707733154296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,10240,0.03332373301188151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,10240,0.03273599942525228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,10240,0.04877333243687947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,1024,0.008920533458391826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,8192,0.023720532655715942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,768,0.007969066500663757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,8192,0.027435733874638872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,8192,0.041092268625895184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,7168,0.021684267123540244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,7168,0.026076799631118773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,6144,0.019099734226862588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,6144,0.0250218669573466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,5120,0.015963733196258545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,5120,0.02315946618715922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,256,0.005897599955399831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,4096,0.013138133287429809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,4096,0.02174933354059855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,4096,0.02267306645711263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,3584,0.011981866757074992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,3584,0.021061333020528157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,3584,0.01993066668510437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,3072,0.010841600100199382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,3072,0.02015893260637919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,3072,0.017615999778111777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,2560,0.00955839951833089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3584,128,0.005293866495291392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,2560,0.01923946738243103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,2048,0.008404266834259034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,2048,0.018781866629918417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,51200,0.21646505991617837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,2048,0.012819199760754903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,1536,0.00883733332157135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,16384,0.07571626504262288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,1536,0.017777067422866822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,1536,0.010599467158317565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,1024,0.0065760001540184024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,1024,0.016797866423924765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,768,0.005243733525276184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,12288,0.0552512009938558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,768,0.01627306640148163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,512,0.004567466676235199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,512,0.015897599856058757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,6144,0.032806400458017984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,256,0.003903999924659729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,7168,0.036534400780995686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,256,0.015188266833623251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,128,0.003472000112136205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,5120,0.028568534056345622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,128,0.01483519971370697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,64,0.003219199925661087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,64,0.01495039959748586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,3072,32,0.003368533402681351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,3072,32,0.0151936004559199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,65536,0.13686827023824055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,768,0.0077237332860628765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,65536,0.1139562686284383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,51200,0.10947733720143635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,65536,0.2513610680898031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,51200,0.09394559860229493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,2560,0.016189866264661155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,16384,0.04042773246765137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,16384,0.041006934642791745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,16384,0.0756661335627238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,12288,0.03258986671765645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,12288,0.05498773256937663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,12288,0.03450666666030884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,10240,0.03154453237851461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,10240,0.04530239899953206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,8192,0.023082667589187623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,8192,0.027739733457565308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,8192,0.039774934450785324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,7168,0.020771199464797975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,7168,0.02638079921404521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,6144,0.018212266763051353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,6144,0.024100265900293984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,1024,0.008505599697430928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,5120,0.013889066378275552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,5120,0.02290453314781189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,512,0.006617600222428639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,4096,0.013814399639765421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,51200,0.2013557275136312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,4096,0.021668267250061036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,3584,0.010445866982142131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,10240,0.028449066480000812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,3584,0.020760534207026164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,3072,0.01125759979089101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,3072,0.01964799960454305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,256,0.005725866556167603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,3072,0.017257599035898845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,3072,128,0.005083733300367991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,2560,0.009795199831326802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,2560,0.01572480003039042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,2560,0.017720532417297364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,2048,0.008411733309427898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,1536,0.006572799881299336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,2048,0.01755946675936381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,1536,0.01727466583251953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,1024,0.005432533224423727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,1024,0.016511999567349753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,1536,0.010451199611028035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,768,0.004309333364168803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,768,0.01577173372109731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,768,0.007609599828720092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,512,0.003835733234882355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,7168,0.03476053476333618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,512,0.015896532932917276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,512,0.006605866551399231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,6144,0.032102400064468385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,256,0.003571200122435888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,128,0.003219199925661087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,256,0.005724800129731497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,256,0.015027200182278952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,64,0.003009066730737686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,64,0.014972800016403198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2560,32,0.0030933332939942675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,32,0.014936533570289613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,5120,0.026409600178400678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,65536,0.1016650676727295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,4096,0.021922133366266885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,51200,0.08540373643239339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,65536,0.23026666641235352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,3584,0.020071466763814293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,16384,0.03377600113550822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,51200,0.08341546853383383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,16384,0.03870720068613688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,2048,0.012482133507728577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,16384,0.0664085348447164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,12288,0.027412267525990804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,12288,0.0312885324160258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,12288,0.04900693496068319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,10240,0.025073067347208662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,10240,0.028217599789301558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,1024,0.008634666601816814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2560,128,0.014937600493431092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,8192,0.018908800681432088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,7168,0.01695466637611389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,8192,0.025711999336878462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,7168,0.024145066738128662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,7168,0.03217493295669556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,6144,0.014846932888031007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,6144,0.02313279906908671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,5120,0.013718400398890176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,65536,0.11344213485717773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,5120,0.021961599588394165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,4096,0.011710932850837708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,5120,0.023240532477696738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,4096,0.019722666343053183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,4096,0.02027413249015808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,3584,0.010372266173362732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,3584,0.019934932390848793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,3584,0.017915733655293784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,3072,0.009335466225941976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,3072,0.015383467078208923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,3072,0.0192522664864858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,2560,0.008361599842707316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,2560,0.018503467241923012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,2560,0.013895466923713684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,2048,0.007865599791208903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,2048,0.017412267128626504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,1536,0.006257066627343495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,1536,0.016523733735084534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2560,128,0.005172266562779745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,1024,0.004894933104515076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,1024,0.01604480048020681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,51200,0.17984213829040527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,768,0.00429013321797053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,768,0.015870933731396995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,512,0.0038165333370367683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,8192,0.035894401868184406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,10240,0.041577601432800294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,512,0.015451733271280924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,512,0.006694399813810985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,256,0.0033749334514141084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,256,0.015191466609636942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,128,0.0031850665807724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,256,0.005672533313433329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,64,0.0029834667841593427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,6144,0.028350933392842607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,128,0.014870400230089823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,2048,32,0.00310506671667099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,64,0.014909866452217101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,2048,32,0.014965333541234336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,65536,0.08911466598510742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,65536,0.09212266604105632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,51200,0.07291413148244222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,51200,0.07354453404744467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,16384,0.02757333318392436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,16384,0.03574719826380412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,2048,0.01160533328851064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,12288,0.02258239984512329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,12288,0.03369813362757365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,1536,0.010048000017801921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,10240,0.0204202671845754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,10240,0.02887786626815796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,10240,0.04416746695836385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,1024,0.008243200182914735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,8192,0.01705706715583801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,8192,0.026150399446487428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,7168,0.01813760002454122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,7168,0.02500586708386739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,768,0.007514666517575581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,6144,0.016511999567349753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,6144,0.023592533667882283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,2048,128,0.00525546669960022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,5120,0.013846400380134582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,5120,0.022241065899531044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,65536,0.23141226768493653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,4096,0.011832533280054729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,51200,0.18601493835449218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,4096,0.01984000007311503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,3584,0.011230933666229247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,3584,0.019033600886662803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,16384,0.06942400137583414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,3584,0.018544000387191773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,3072,0.009733333190282186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,3072,0.01881706714630127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,2560,0.008982400099436443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,2560,0.01874026656150818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,12288,0.05094720125198364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,2048,0.007579733431339264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,2048,0.017646932601928712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,8192,0.03688640197118123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,1536,0.006244266529877981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,1536,0.016964266697565712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,7168,0.03371520042419433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,1024,0.005127466718355815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,1024,0.016294399897257485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,1024,0.008175999919573466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,768,0.00462719996770223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,6144,0.03001813292503357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,768,0.01575040022532145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,5120,0.0265173335870107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,512,0.00413973331451416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,512,0.016074666380882265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,4096,0.020910932620366415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,256,0.0034474665919939675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,256,0.014903466900189719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,256,0.0054730668663978575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,128,0.0032085334261258446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,3072,0.015852800011634825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,128,0.01480959951877594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,64,0.002958933264017105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,2560,0.01406826674938202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1536,32,0.0031167998909950255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,32,0.014986667037010192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,2048,0.01181653340657552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,65536,0.06091093222300211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,65536,0.07605546315511068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,51200,0.04967679977416992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,1536,0.010014933347702027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,51200,0.06274880170822143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,16384,0.019568000237147012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,16384,0.030115199089050294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,12288,0.015367466211318969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,12288,0.026850134134292603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1536,64,0.014839466412862143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,10240,0.014838400483131408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,10240,0.02414399981498718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,10240,0.0379530668258667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,8192,0.013025066256523133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,8192,0.023154133558273317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,768,0.007436800003051758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,7168,0.01220906674861908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,7168,0.022574933369954427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,512,0.0063701331615448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,6144,0.01120853324731191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,6144,0.02140480081240336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1536,128,0.004955733319123586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,5120,0.010264533758163451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,5120,0.02072319984436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,65536,0.2030837376912435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,4096,0.009542399644851684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,4096,0.018761599063873292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,51200,0.16110933621724446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,3584,0.008813866972923278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,3584,0.01844693422317505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,3584,0.017113600174585977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,3072,0.007944533228874206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,3072,0.018205867211023966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,3072,0.014354133605957031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,2560,0.007348266740640004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,2560,0.01747200091679891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,2560,0.013123200337092081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,2048,0.006306133170922597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,12288,0.04570879936218262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,2048,0.01702079971631368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,2048,0.011212799946467083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,1536,0.005388799806435903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,1536,0.016354133685429893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,1024,0.004584533472855886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,1024,0.015864533185958863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,8192,0.03370560010274251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,768,0.004053333401679992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,768,0.015925332903862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,768,0.007333333293596904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,512,0.003772799919048945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,16384,0.05905813376108805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,7168,0.029385600487391157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,512,0.015094400445620219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,256,0.003302400062481562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,256,0.014959999918937683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,128,0.0030122667551040648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,128,0.014784000317255654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,128,0.005081599950790406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,64,0.002885333448648453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,64,0.01476693352063497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,1024,32,0.0029088000456492106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,1024,32,0.014854400356610616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,65536,0.04994879961013794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,65536,0.07073173522949219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,6144,0.02608533302942912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,5120,0.022829866409301756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,51200,0.0425002654393514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,51200,0.05885333220163981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,16384,0.018427733580271402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,16384,0.030080000559488933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,512,0.0063391998410224915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,12288,0.015130666891733804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,4096,0.018785067399342856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,12288,0.026683733860651655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,10240,0.013684266805648803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,10240,0.02423786719640096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,10240,0.03290133277575175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,8192,0.014602667093276978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,8192,0.022454400857289634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,1536,0.009522133072217305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,7168,0.012541866302490235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,7168,0.022116265694300332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,1024,0.008087466657161712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,7168,0.024959999322891235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,6144,0.011758933464686077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,6144,0.02137813369433085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,1024,256,0.005418666700522105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,5120,0.010654933253924052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,5120,0.01989013353983561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,5120,0.018669867515563966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,4096,0.00918293297290802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,4096,0.018927999337514243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,65536,0.18475947380065919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,3584,0.008403199911117553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,51200,0.14632533391316732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,3584,0.018127999703089395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,3072,0.007188266515731812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,3072,0.017933867375055947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,16384,0.05137279828389486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,3072,0.012475732962290447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,2560,0.006404266754786174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,2560,0.011201066772143047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,2560,0.017219199736913045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,2048,0.005611733098824819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,2048,0.016908800601959227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,12288,0.03863786856333415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,1536,0.004939733445644379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,1536,0.016669867436091106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,1024,0.004137599964936575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,1024,0.015620266397794088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,1024,0.007170133292675018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,768,0.0038880000511805216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,768,0.015662933389345803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,8192,0.027419734001159667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,512,0.0036256000399589538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,512,0.015698132912317912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,512,0.0059125334024429325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,256,0.0032149332265059153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,256,0.014546133081118264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,128,0.002995199958483378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,6144,0.022066134214401244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,128,0.004931200047334035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,128,0.014757333199183145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,64,0.002812800059715907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,768,32,0.0028533334533373516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,64,0.01479573349157969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,768,32,0.014829867084821067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,65536,0.036807465553283694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,65536,0.06392746766408285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,51200,0.03041599988937378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,51200,0.05371520121892294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,4096,0.015732266505559287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,16384,0.01165120005607605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,16384,0.026549333333969118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,3584,0.014494933684666953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,12288,0.010519466797510783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,12288,0.023643734057744344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,2048,0.009715200463930766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,12288,0.03159466584523519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,10240,0.012444800138473511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,1536,0.008635733524958293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,10240,0.022915200392405192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,10240,0.026255999008814496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,8192,0.010674132903416952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,8192,0.022123734156290688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,7168,0.010010666648546855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,7168,0.02171199917793274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,6144,0.00904746651649475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,6144,0.020869332551956176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,768,0.006604800124963124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,6144,0.018093866109848023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,5120,0.008141866823037466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,5120,0.0197269340356191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,4096,0.008477866649627686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,4096,0.018563199043273925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,768,256,0.005193600058555603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,3584,0.0077567999561627705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,3584,0.018053332964579262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,65536,0.14393173853556315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,3072,0.007005866865317028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,3072,0.017914666732152303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,3072,0.010097066561381023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,2560,0.006309333443641663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,51200,0.11490879853566487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,2560,0.017413334051767985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,2048,0.005566933254400889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,16384,0.04174720048904419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,2048,0.016976000865300496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,1536,0.0049002667268117275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,2048,0.008205866813659668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,1536,0.0074890668193499255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,1536,0.01639359990755717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,1024,0.004199466605981191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,1024,0.0064298664530118305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,1024,0.015709867080052696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,768,0.0037632000943024955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,768,0.005982933441797892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,768,0.015369600057601929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,512,0.003522133330504099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,512,0.015219199657440185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,512,0.005498666564623515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,256,0.003202133377393087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,256,0.0050794666012128195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,256,0.014717866977055868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,128,0.002902399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,128,0.01518933375676473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,128,0.00484799991051356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,64,0.002863999952872594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,64,0.01454080045223236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,512,32,0.0028768000503381092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,512,32,0.01460693379243215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,65536,0.025887999931971235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,8192,0.02262079914410909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,7168,0.01964906652768453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,65536,0.0576533317565918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,65536,0.10425066947937012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,51200,0.021409066518147786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,51200,0.04533439874649048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,16384,0.009866666793823243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,2560,0.00932373305161794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,51200,0.08204267024993897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,12288,0.009335466225941976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,16384,0.023884799083073935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,16384,0.027980800469716387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,10240,0.009790933132171631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,12288,0.02137920061747233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,10240,0.021421867609024047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,8192,0.008113066852092742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,10240,0.018322134017944337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,7168,0.007795199751853943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,8192,0.01971199909845988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,7168,0.020106667280197145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,6144,0.007315200070540111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,6144,0.01945706605911255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,4096,0.01276586651802063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,6144,0.012853333353996277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,5120,0.006738133231798808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,5120,0.01947093407313029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,4096,0.006306133170922597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,3584,0.011502933502197266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,4096,0.01871466636657715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,3584,0.00673280010620753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,3584,0.01807360053062439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,3584,0.009729066491127014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,3072,0.006386133531729381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,3072,0.01761173407236735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,3072,0.008697600166002909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,2560,0.006155733267466227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,2560,0.017056000232696534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,2560,0.008178133269151051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,2048,0.005452799797058106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,12288,0.02172799905141195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,512,5120,0.014937600493431092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,2048,0.017013333241144814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,2048,0.007221333185831706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,1536,0.004741333425045013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,1536,0.00691840002934138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,1024,0.004070399949947992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,1024,0.015290666619936624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,1024,0.005922133227189382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,768,0.003798400113979975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,768,0.015607466300328573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,768,0.00555733342965444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,512,0.0034656000634034475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,512,0.015016532937685647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,512,0.005310933291912079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,256,0.0030623999734719592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,256,0.014712533354759217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,256,0.0049216002225875854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,128,0.0029109333952267963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,128,0.014571733276049294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,128,0.004648533463478088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,64,0.0027637332677841187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,256,32,0.00271573339899381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,64,0.014553599556287131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,32,0.014513066411018372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,65536,0.01755519906679789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,65536,0.051319468021392825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,65536,0.09980586369832357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,51200,0.013022933403650919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,51200,0.0395306666692098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,16384,0.008633599678675333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,16384,0.0226197342077891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,16384,0.026529065767923993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,12288,0.021062399943669638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,256,1536,0.016404267152150473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,10240,0.0071712002158164975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,10240,0.02051946719487508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,10240,0.01810773412386576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,8192,0.006662400066852569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,8192,0.01560640037059784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,8192,0.015253333250681558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,8192,0.019437867403030395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,7168,0.01399679978688558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,7168,0.006378666559855144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,6144,0.006156800190607706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,7168,0.020029866695404054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,6144,0.012743467092514038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,6144,0.019397334257761637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,5120,0.006495999793211619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,5120,0.019610667228698732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,5120,0.011313066879908244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,4096,0.006087466577688853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,4096,0.018458666404088338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,4096,0.010120532910029094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,12288,0.007709866762161255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,3584,0.006534400085608165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,3584,0.00969599982102712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,3584,0.01794346570968628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,3072,0.006241066753864289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,3072,0.017259732882181803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,3072,0.008403199911117553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,2560,0.01709653337796529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,2560,0.008050133287906647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,2048,0.0054293334484100345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,2048,0.016748799880345663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,7168,0.013797332843144735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,1536,0.0046847999095916745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,1536,0.016153599818547568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,1536,0.006866133213043213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,256,4096,0.010223999619483948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,1024,0.004136533290147781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,1024,0.015526400009791056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,768,0.0036469332873821257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,768,0.015146666765213012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,2560,0.006054399907588959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,51200,0.07862826983133951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,512,0.0033386667569478357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,512,0.015413332978884378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,256,0.0029696000119050344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,12288,0.021079466740290324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,256,0.014698666334152222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,128,0.002812800059715907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,256,0.004940799872080485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,128,0.014659200112024942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,64,0.0027295999228954316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,64,0.01442346672217051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,128,32,0.002721066772937775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,128,32,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,65536,0.01130986710389455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,65536,0.047202134132385255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,51200,0.010201600193977357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,51200,0.03731840054194133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,16384,0.006853333115577698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,5120,0.01137600044409434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,12288,0.006921599805355072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,12288,0.021217066049575805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,10240,0.006471466521422069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,10240,0.020348799228668214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,8192,0.0063509335120519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,7168,0.006188799937566122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,8192,0.01938026746114095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,7168,0.020040533939997354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,6144,0.0060032000144322716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,5120,0.006340266764163971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,6144,0.01962453325589498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,5120,0.019317332903544107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,4096,0.018595200777053834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,3584,0.0064522668719291685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,3584,0.018049067258834837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,3072,0.006167466441790262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,3072,0.01741546591122945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,2560,0.006011733412742614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,2048,0.007167999943097432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,2048,0.005330133438110352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,2560,0.017062399784723917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,1536,0.004633600016434988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,2048,0.01665066679318746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,1024,0.003970133264859518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,1536,0.016177067160606386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,768,0.0036159999668598174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,1024,0.015635200341542563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,512,0.0034944000343481696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,768,0.015281066298484802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,256,0.0029365333418051405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,512,0.014923733472824097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,128,0.002850133428970973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,256,0.014588800072669984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,64,0.0026528000831604003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,128,0.014572800199190775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,32,0.0026911998788515727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,64,0.014523733655611673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,65536,0.010318932930628459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,32,0.014528000354766845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,51200,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,65536,0.04626986583073934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,16384,0.006906666855017345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,768,0.005621333420276642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,12288,0.00690773328145345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,16384,0.022259199619293214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,10240,0.006433066725730896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,12288,0.020837332804997763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,8192,0.006278400123119354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,10240,0.020138667027155558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,64,4096,0.0060127998391787205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,7168,0.0061930666367212926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,8192,0.019236266613006592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,7168,0.020198400815327963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,6144,0.005989333490530649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,512,0.005245866874853769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,5120,0.0063701331615448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,4096,0.0059690664211908976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,6144,0.019197867314020792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,5120,0.019805866479873657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,3584,0.0063498665889104204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,4096,0.018613332509994508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,3072,0.006077866752942403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,3584,0.017884800831476845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,128,0.00487360010544459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,2560,0.0059797331690788266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,3072,0.017554134130477905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,2048,0.005425066749254862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,2560,0.016938666502634682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,1536,0.004659200211366018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,2048,0.01647040049235026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,1024,0.003996799886226654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,1536,0.015998933712641397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,768,0.003589333345492681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,1024,0.015870933731396995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,512,0.0033962666988372804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,768,0.015851733088493348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,256,0.002961066613594691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,128,0.0028021333118279776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,256,0.014630400141080222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,64,0.002717866748571396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,128,0.014574933052062988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,161,32,32,0.002699733277161916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,64,0.014493866761525472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,32,0.01458453337351481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,64,16384,0.022667733828226726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,51200,0.036039467652638754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,161,32,512,0.014913066228230795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,16384,0.7729621251424154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,16384,0.41884905497233077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,161,128,1024,0.006017066538333893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,12288,0.32113173802693684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,12288,0.5905248006184896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,51200,1.2448010762532553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,16384,0.7020714441935222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,12288,0.5430453618367512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,10240,0.2707765261332194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,10240,0.5055338541666667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,8192,0.22258240381876626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,8192,0.396671994527181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,51200,2.4119552612304687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,10240,0.48362986246744794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,7168,0.3479455947875977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,7168,0.19717760086059571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,51200,2.324095916748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,8192,0.40331201553344725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,6144,0.30395946502685545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,7168,0.3560138702392578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,5120,0.25362133979797363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,5120,0.14885867436726888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,6144,0.3054837226867676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,4096,0.2073919932047526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,4096,0.12387733459472657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,6144,0.19682666460673015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,4096,0.19659306208292643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,3584,0.18175253868103028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,3584,0.11235307057698567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,5120,0.2536191940307617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,3072,0.1582709312438965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,3072,0.09957119623819986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,2560,0.13324800332387288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,3584,0.17513279914855956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,2560,0.08793919881184896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,3072,0.14876799583435057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,1536,0.08588266372680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,2048,0.07457173665364583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,2048,0.10140906969706218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,1536,0.06137066682179769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,1024,0.0681717316309611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,1024,0.04824533462524414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,1536,0.07672426700592042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,768,0.04842453400293986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,1024,0.05379626750946045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,768,0.04174506664276123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,512,0.036076800028483076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,256,0.021911466121673585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,512,0.03534719944000244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,512,0.03445440133412679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,256,0.02998720010121663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,768,0.047134931882222494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,128,0.013301333785057068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,64,0.010584533214569092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,32,0.010107733805974324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,128,0.02320746580759684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,32,0.023254400491714476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,64,0.02287999987602234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,2048,0.11421546936035157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,2560,0.12629120349884032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,256,0.0264789342880249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,128,0.02719893256823222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,16384,0.33497066497802735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,16384,0.6203477223714192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,65536,1.2745760599772136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,51200,1.9405418395996095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,51200,1.000879987080892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,12288,0.47115093866984054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,12288,0.29738559722900393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,65536,2.455042012532552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,10240,0.2290485382080078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,10240,0.3975402514139811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,12288,0.46404053370157883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,10240,0.39842027028401694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,51200,1.8697429656982423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,8192,0.366215451558431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,7168,0.2760522524515788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,8192,0.20319040616353354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,7168,0.18291840553283692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,8192,0.3412896156311035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,6144,0.14335573514302571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,6144,0.24172372817993165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,7168,0.3010858535766602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,5120,0.12406613032023113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,5120,0.20633920033772787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,4096,0.1640127976735433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,4096,0.10406293074289959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,3584,0.14591147104899088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,3584,0.0947861353556315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,3072,0.12641706466674804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,3584,0.14653013547261556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,3072,0.08526293436686197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,3072,0.1212725321451823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,2560,0.10823146502176921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,2560,0.07431999842325845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,2560,0.10685973167419434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,2048,0.08859840234120687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,2048,0.0855072021484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,1536,0.07084480126698812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,1536,0.053608532746632895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,1536,0.06372479995091757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,16384,0.5850869496663411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,1024,0.04997013409932454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,1024,0.04541226625442505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,768,0.04031360149383545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,768,0.037536001205444335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,65536,2.3920799255371095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,768,0.03504426479339599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,512,0.03123840093612671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,512,0.03297066688537598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,512,0.026477867364883424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,256,0.026393600304921466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,256,0.018373332420984902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,128,0.01309333344300588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,128,0.02256960074106852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,2048,0.0655893325805664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,64,0.010005333026250203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,128,0.01829866568247477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,51200,32,0.009545600414276123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,64,0.021744000911712646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,51200,32,0.021991467475891112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,6144,0.25189119974772134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,5120,0.20962559382120766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,4096,0.16698986689249676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,65536,0.4196053187052409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,65536,0.7968661626180013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,16384,0.19643093744913737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,51200,0.33115946451822914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,51200,0.6278858820597331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,16384,0.13584319750467938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,51200,0.5747925440470378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,12288,0.10686826705932617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,10240,0.12428053220113118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,12288,0.14660372734069824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,10240,0.08201066652933756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,8192,0.09924800395965576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,10240,0.12727466424306233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,8192,0.06897173722585043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,8192,0.10494506359100342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,7168,0.06294613281885783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,7168,0.08799786567687988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,1024,0.0452458659807841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,7168,0.09421439965566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,6144,0.07736746470133463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,6144,0.05820480187733969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,5120,0.06529599825541178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,5120,0.050785064697265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,4096,0.05347733497619629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,4096,0.04381973346074422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,4096,0.05494079987208048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,3584,0.047729067007700604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,3584,0.04103786547978719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,3584,0.04967999855677287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,3072,0.037581864992777506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,12288,0.14625919659932454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,3072,0.04242986838022868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,2560,0.038424531618754074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,51200,256,0.020499199628829956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,2560,0.03404159943262736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,2560,0.03728426694869995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,2048,0.03004693388938904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,1536,0.024242132902145386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,1536,0.02616320053736369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,1536,0.023515733083089192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,1024,0.017064533631006875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,1024,0.02240426739056905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,3072,0.04186240037282308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,768,0.013184000054995218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,768,0.020719999074935914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,65536,0.695747184753418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,512,0.011096533139546711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,2048,0.03022506634394328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,512,0.01962239940961202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,2048,0.03061760067939758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,256,0.0071712002158164975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,512,0.013220266501108805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,256,0.010813867052396137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,256,0.017010132471720375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,16384,0.18824960390726725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,128,0.005187200009822845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,128,0.01649493376413981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,128,0.009649067123730978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,32,0.00505920002857844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,64,0.016693333784739174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,32,0.01732053359349569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,6144,0.08092586994171143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,5120,0.06635946830113729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,65536,0.34933865865071617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,65536,0.5967605590820313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,64,0.004588800172011057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,51200,0.46975040435791016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,51200,0.2784213383992513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,65536,0.6277215957641602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,51200,0.4901877403259277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,16384,0.16449066797892253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,16384,0.11090986728668213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,16384,0.16327679951985677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,12288,0.13114346663157145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,12288,0.0841386636098226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,10240,0.09499839941660562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,10240,0.06757866541544597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,10240,0.1118016004562378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,8192,0.07642133235931396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,8192,0.09524052937825521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,8192,0.05694506565729777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,7168,0.06867199738820394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,6144,0.05986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,7168,0.05229973395665487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,1024,0.018563199043273925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,6144,0.0476309339205424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,5120,0.05101439952850342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,4096,0.04114346504211426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,5120,0.057914666334788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,4096,0.03706026474634806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,4096,0.04874453147252401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,3584,0.03672426541646322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,3584,0.034924801190694174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,768,0.015191466609636942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,3584,0.044360534350077314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,3072,0.03254400094350179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,3072,0.036448001861572266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,2560,0.027881600459416706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,3072,0.032092799743016556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,2560,0.028597333033879596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,2048,0.0232149342695872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,2048,0.025560534000396727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,2048,0.027130667368570966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,1536,0.018556799491246542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,1536,0.023042132457097374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,1024,0.01267413298288981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,1024,0.020860799153645835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,1024,0.015685333808263143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,768,0.010502400000890096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,768,0.013799466689427695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,768,0.019717333714167277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,512,0.008308266599973042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,512,0.017180800437927246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,256,0.005226666728655497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,256,0.01662720044453939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,256,0.008717866738637288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,5120,0.04226453304290771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,128,0.004466133316357931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,128,0.015964800119400026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,128,0.007879466811815897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,64,0.0039711999396483105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,64,0.01618346671263377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,32,0.004424533247947693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,32,0.01656426688035329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,1536,0.020890667041142782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,65536,0.31089067459106445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,12288,0.12655680179595946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,51200,0.40580053329467775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,7168,0.08359466393788656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,6144,0.07039039929707845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,51200,0.24967573483784994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,16384,0.1479317347208659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,16384,0.08723413149515788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,12288,0.0985706647237142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,12288,0.07025492986043294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,16384,0.1544256051381429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,51200,0.4557632128397624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,10240,0.08258240222930908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,10240,0.06207786798477173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,8192,0.06658133268356323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,2560,0.032257066170374556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,65536,0.5131807963053385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,8192,0.08700479666392008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,8192,0.05115840037663778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,7168,0.05986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,7168,0.0775381326675415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,6144,0.052355198065439854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,7168,0.051482665538787845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,6144,0.043075199921925864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,6144,0.06675626436869303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,5120,0.038761599858601885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,5120,0.04425493478775024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,512,0.01065066655476888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,4096,0.0339903990427653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,4096,0.046241064866383866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,3584,0.0318997323513031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,3584,0.04199039936065674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,3584,0.03206826647122701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,3072,0.02834879954655965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,3072,0.028961066404978437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,2560,0.024258132775624594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,2560,0.02664320071538289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,2560,0.030402133862177532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,2048,0.020406399170557657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,2048,0.024120533466339113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,2048,0.025463465849558515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,1536,0.015703466534614564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,1536,0.02210986614227295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,4096,0.03535786469777425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,1024,0.011341866850852967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,1024,0.020198400815327963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,65536,0.5758250554402669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,768,0.009795199831326802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,768,0.019602133830388387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,768,0.01267093320687612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,512,0.00787306676308314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,512,0.017467733224232992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,12288,0.12107733090718586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,256,0.0052373334765434265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,10240,0.10540160338083904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,256,0.01663146714369456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,128,0.004488533238569895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,256,0.007749333480993907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,128,0.015915733575820924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,64,0.0040501333773136135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,64,0.015955199797948204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,32,0.004183466732501984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,32,0.015957333644231162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,5120,0.055829334259033206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,65536,0.24438400268554689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,65536,0.4138250668843587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,3072,0.03478720188140869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,65536,0.364629332224528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,16384,0.09774400393168131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,51200,0.1968597412109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,1536,0.01960960030555725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,51200,0.33034772872924806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,51200,0.2966090520222982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,12288,0.07574400107065836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,16384,0.10298986434936523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,12288,0.0642677346865336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,10240,0.06401066780090332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,12288,0.07989866733551025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,8192,0.05145279963811239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,10240,0.0515008012453715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,8192,0.04435520172119141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,8192,0.05385493437449137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,7168,0.0459114670753479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,7168,0.04071360031763713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,6144,0.04072320063908895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,6144,0.03761279980341593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,6144,0.04310186703999837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,5120,0.03477973143259684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,1024,0.01474453310171763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,5120,0.03362026611963908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,5120,0.03559360106786092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,4096,0.029908267656962077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,3584,0.02549546758333842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,3584,0.027756800254185993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,3584,0.027672533194224042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,3072,0.022819199164708457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,3072,0.02558293342590332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,512,0.010550399621327717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,3072,0.023692800601323446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,2560,0.019488000869750978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,2560,0.02409279942512512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,2560,0.021088000138600668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,2048,0.016089600324630738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,2048,0.02226240038871765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,16384,0.07248319784800211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,1536,0.012495999534924824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,1536,0.014802133043607077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,1536,0.02072426676750183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,1024,0.009435733159383137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,1024,0.0118367999792099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,768,0.00796693315108617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,128,0.007107200225194295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,768,0.0176256000995636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,768,0.010437333583831787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,512,0.0066431999206542965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,512,0.01662720044453939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,512,0.008583466211954754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,256,0.004331733286380768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,256,0.0157258669535319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,128,0.003949866692225138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,4096,0.02873706618944804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,128,0.015597866972287497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,128,0.00681386689345042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,64,0.003610666592915853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,64,0.015643733739852905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,32,0.003999999910593033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,32,0.015990400314331056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,65536,0.3561984062194824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,1024,0.01893333395322164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,65536,0.23156800270080566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,10240,0.06792853673299154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,7168,0.04941973288853963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,51200,0.17801493008931477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,51200,0.2833919843037923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,16384,0.08622933228810628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,4096,0.03014506697654724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,51200,0.27343358993530276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,16384,0.0732096036275228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,12288,0.06789120038350424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,16384,0.09402026335398356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,12288,0.07343893051147461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,12288,0.054406400521596274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,10240,0.05673280159632364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,10240,0.04782079855600993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,8192,0.04581226507822673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,8192,0.04219839970270793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,10240,0.06286400159200033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,7168,0.04114986658096313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,8192,0.050329601764678954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,7168,0.03826773166656494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,6144,0.03647893269856771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,6144,0.0353216012318929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,6144,0.03904106616973877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,5120,0.03174933393796285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,2048,0.01776319940884908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,4096,0.025484800338745117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,4096,0.027689599990844728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,4096,0.027803732951482134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,3584,0.022949333985646567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,3584,0.025917865832646686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,3072,0.02062293291091919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,3072,0.024473599592844644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,3072,0.021848533550898233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,2560,0.017571200927098594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,256,0.007608533402283986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,2560,0.02318506638209025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,2560,0.019220266739527384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,2048,0.014139733711878457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,2048,0.01629973351955414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,2048,0.021707733472188316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,1536,0.011355732878049214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,1536,0.020439465840657554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,1024,0.008961066603660583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,1024,0.018447999159495035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,1024,0.01093226671218872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,768,0.007642666498819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,768,0.017190400759379068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,512,0.005758933226267497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,512,0.016595199704170227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,512,0.007785599927107494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,5120,0.031061333417892457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,256,0.004298666616280874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,5120,0.03316906690597534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,256,0.006549333532651265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,256,0.01581546664237976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,128,0.015593600273132325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,128,0.005914666752020518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,64,0.003748266647259394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,32,0.0037994667887687682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,64,0.01627306640148163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,32,0.015656532843907674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,65536,0.20315839449564616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,65536,0.30677973429361977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,65536,0.33986988067626955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,51200,0.24375999768575035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,65536,0.31682027180989586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,51200,0.16474134127298992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,16384,0.08957119782765707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,16384,0.0608842651049296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,16384,0.08741333484649658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,12288,0.06814613342285156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,128,0.003765333443880081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,51200,0.25797333717346194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,12288,0.050316798686981204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,10240,0.058747732639312746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,7168,0.044240001837412515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,8192,0.04083520174026489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,8192,0.03818560043970744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,8192,0.047975468635559085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,7168,0.036882134278615315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,7168,0.03628160158793132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,7168,0.04165120124816894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,6144,0.03266666730244954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,6144,0.03292906681696574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,3584,0.025665066639582318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,6144,0.03643413384755452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,5120,0.02802773316701253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,4096,0.022844799359639487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,5120,0.029340799649556475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,5120,0.03152533372243245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,3584,0.020778665939966835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,4096,0.02622399926185608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,3584,0.024872533480326333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,3584,0.024210133155186973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,3072,0.01840959986050924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,3072,0.023780266443888344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,1536,0.01344000001748403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,3072,0.020838399728139244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,2560,0.02248959938685099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,2560,0.018823466698328652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,2048,0.012857600053151449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,768,0.009128533800443013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,2048,0.020977065960566203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,2048,0.015780267119407655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,1536,0.010551466544469198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,1536,0.013025066256523133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,1536,0.01994026700655619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,1024,0.008377599716186523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,10240,0.04491839806238811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,1024,0.017425066232681273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,1024,0.01011199951171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,768,0.01688106656074524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,768,0.008541867136955261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,512,0.0051701332132021586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,512,0.016432000199953715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,512,0.007640533149242401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,256,0.004208000004291534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,256,0.01579200029373169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,256,0.006533333162466686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,128,0.0038592000802357995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,128,0.015275733669598899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,128,0.005949866771697998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,64,0.003522133330504099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,32,0.003700266778469086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,64,0.015736533204714458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,32,0.015457066893577575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,2560,0.015719466408093772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,65536,0.24870293935139975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,65536,0.17508479754130046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,51200,0.19784746170043946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,51200,0.14037334124247233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,65536,0.2946314811706543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,16384,0.07416319847106934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,768,0.007289599875609081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,16384,0.06085333426793417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,16384,0.08349546591440836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,51200,0.23414079348246256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,12288,0.05925973256429037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,12288,0.04674559831619263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,10240,0.05242133140563965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,12288,0.0687455972035726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,10240,0.04151573181152344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,10240,0.05883306662241618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,8192,0.041934935251871745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,7168,0.03295786579449971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,8192,0.035613866647084554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,7168,0.03370346625645955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,6144,0.02919999957084656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,7168,0.03980266650517782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,5120,0.025217066208521526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,6144,0.030199466149012248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,5120,0.027962666749954224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,4096,0.02026240030924479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,4096,0.0252895991007487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,4096,0.025613866249720257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,4096,0.02694186568260193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,3584,0.018685867389043175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,3584,0.024317866563796996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,3072,0.016219733158747356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,3072,0.02304640014966329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,2560,0.01372160017490387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,2560,0.021980800231297812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,12288,0.06680320103963217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,2048,0.0116074671347936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,8192,0.044698667526245114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,2048,0.020555732647577922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,2048,0.014847999811172486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,1536,0.009934932986895243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,1536,0.019318399826685588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,1024,0.007910400132338206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,1024,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,1024,0.009458133578300476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,768,0.0068341334660847975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,768,0.01679253379503886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,10240,0.05621546506881714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,512,0.005049600203831991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,512,0.016526933511098227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,5120,0.029048534234364827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,6144,0.03527466853459676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,256,0.004203733305136363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,256,0.00618453323841095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,256,0.015799466768900552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,128,0.0038015998899936674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,128,0.015522133310635886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,64,0.0034826666116714476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,3584,0.02290560007095337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,32,0.0034677334129810332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,32,0.015356799960136414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,3072,0.020056533813476562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,65536,0.20372160275777182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,2560,0.017562667528788246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,65536,0.15065280596415204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,51200,0.1636031945546468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,51200,0.12383786837259929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,1536,0.012179199854532878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,51200,0.21301867167154948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,16384,0.05844586690266927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,64,0.015340800086657206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,768,0.008168533444404602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,16384,0.05295039812723795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,12288,0.04712959925333659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,12288,0.04699733257293701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,10240,0.04106880029042562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,10240,0.036696533362070724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,8192,0.03449493249257406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,8192,0.03109546701113383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,512,0.007102933526039123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,7168,0.033522133032480875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,128,0.0055861334005991616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,7168,0.02865920066833496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,7168,0.035051735242207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,6144,0.02945599953333537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,6144,0.026627200841903686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,5120,0.020869332551956176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,5120,0.02503146727879842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,65536,0.27506987253824866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,4096,0.01576640009880066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,4096,0.022758400440216063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,4096,0.022706133127212525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,3584,0.014082133769989014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,3584,0.021501866976420085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,3584,0.021217066049575805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,3072,0.01295146644115448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,3072,0.018204800287882485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,3072,0.021081600586573282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,2560,0.01084266702334086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,2560,0.020122667153676353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,2560,0.016223999857902526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,2048,0.009399466713269551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,2048,0.01914986570676168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,1536,0.007935999830563863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,1536,0.017191465695699057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,12288,0.05791039864222208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,16384,0.07460479736328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,1024,0.006426666676998138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,10240,0.049149866898854574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,1024,0.016826667388280234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,1024,0.00900266667207082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,768,0.00528106689453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,768,0.008193066716194153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,768,0.016302933295567833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,512,0.004458666841189066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,8192,0.04023786783218384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,256,0.0038346665600935614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,512,0.015661866466204325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,128,0.0034741332133611047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,256,0.006340266764163971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,256,0.015412267049153647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,128,0.015080533425013223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,64,0.015140266219774882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,32,0.0034613333642482757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,32,0.015270400047302245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,65536,0.18101226488749186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,65536,0.14028693834940592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,6144,0.031126399834950764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,5120,0.026459733645121258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,512,0.007203199962774913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,51200,0.1431850592295329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,64,0.0032405334214369455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,51200,0.11316693623860677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,51200,0.21410667101542152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,16384,0.07516480286916097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,16384,0.049670398235321045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,12288,0.042423466841379806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,10240,0.03687253395716349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,12288,0.043044265111287436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,10240,0.03351893424987793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,10240,0.05168426831563314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,8192,0.031150933106740313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,8192,0.029289599259694415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,7168,0.024216532707214355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,8192,0.041605333487192794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,7168,0.027451733748118084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,7168,0.03669439951578776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,6144,0.02874133388201396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,6144,0.025601067145665485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,5120,0.017772799730300902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,5120,0.023492266734441124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,2048,0.01390720009803772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,5120,0.027133866151173906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,4096,0.01416213313738505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,1536,0.011678933103879293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,4096,0.02208426594734192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,4096,0.02355733315149943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,3584,0.020602667331695558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,3584,0.020730666319529214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,3072,0.011474133531252543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,3072,0.020451200008392335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,16384,0.05181013345718384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,2560,0.010090667009353637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,2560,0.019697066148122153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,2048,0.00872320036093394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,128,0.005820799867312113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,2048,0.017897599935531618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,3584,0.012883200248082479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,1536,0.0076330666740735366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,1536,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,1536,0.010869333148002624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,1024,0.005393066505591074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,1024,0.01607253352801005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,65536,0.2736938794453939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,1024,0.008871466914812724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,768,0.0046186665693918865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,768,0.015997866789499916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,512,0.004070399949947992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,768,0.007540266712506611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,512,0.00651093324025472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,512,0.015689599514007568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,256,0.003664000084002813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,12288,0.05941546758015951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,256,0.005905066430568695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,128,0.0032885332902272543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,128,0.014988799889882406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,128,0.005456000069777171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,64,0.003409066547950109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,64,0.015094400445620219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,32,0.0033439998825391137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,32,0.015105066696802774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,6144,0.03240106701850891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,65536,0.15851945877075196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,65536,0.1285322666168213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,51200,0.1287168025970459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,65536,0.2556330680847168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,51200,0.10531413555145264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,16384,0.0498634656270345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,16384,0.04623253345489502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,16384,0.07309546470642089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,12288,0.03776426712671916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,51200,0.218941863377889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,3072,0.01845759948094686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,12288,0.039242664972941085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,10240,0.0330570658047994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,10240,0.05018986860911051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,8192,0.027754666407903035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,256,0.015324800213177999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,8192,0.03041706681251526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,7168,0.025849600632985432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,7168,0.02581546703974406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,2560,0.016437333822250367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,7168,0.034670933087666826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,6144,0.0254037340482076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,6144,0.025224532683690386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,5120,0.020258132616678873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,2048,0.013522133231163025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,4096,0.01853653391202291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,5120,0.026781866947809856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,3584,0.012000000476837159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,4096,0.021565866470336915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,3584,0.020899200439453126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,3072,0.010792533556620281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,3072,0.020080000162124634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,3072,0.01781439979871114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,2560,0.009459199508031209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,2560,0.01889280080795288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,2560,0.016244266430536905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,2048,0.00827519992987315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,2048,0.017511467138926186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,10240,0.035742934544881186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,1536,0.007214933137098948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,1536,0.01641706625620524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,5120,0.02283733288447062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,1024,0.005138133466243744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,1024,0.016313599546750386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,1024,0.008404266834259034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,768,0.0044938668608665465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,768,0.016099199652671814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,768,0.007228800157705943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,512,0.004010666658480962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,512,0.015596800049146018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,512,0.006493866443634033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,256,0.003621333340803782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,256,0.015131733814875283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,256,0.005676800012588501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,128,0.0032320000231266023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,128,0.015128533045450846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,128,0.005053866902987162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,64,0.003253333270549774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,64,0.01520746648311615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,32,0.003320533285538355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,12288,0.05761066675186157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,32,0.015178666512171427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,65536,0.1359615961710612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,65536,0.11369386514027913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,6144,0.032009599606196086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,4096,0.022062933444976805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,51200,0.09252693653106689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,3584,0.020517333348592123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,16384,0.04019733270009358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,16384,0.042933332920074466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,51200,0.1962015946706136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,12288,0.032440533240636185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,12288,0.036627201239267985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,2048,0.013224533200263977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,10240,0.028892799218495684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,10240,0.03257173299789429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,10240,0.048979198932647704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,1536,0.010711466272672016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,8192,0.03918186823527019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,8192,0.02858346700668335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,7168,0.021288534005482994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,8192,0.03814719915390015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,7168,0.02760319908459981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,7168,0.03405439853668213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,6144,0.021473066012064616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,6144,0.025868799289067584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,6144,0.031082665920257567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,5120,0.01602133313814799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,5120,0.022525866826375328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,5120,0.025384533405303954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,4096,0.014986667037010192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,4096,0.020987733205159505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,4096,0.022603732347488404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,3584,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,3584,0.020398932695388793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,3072,0.01251200040181478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,3072,0.019638399283091225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,8192,0.024422399202982583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,2560,0.008935466408729553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,51200,0.10409279664357503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,2560,0.018629332383473717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,2560,0.015396266182263692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,2048,0.007771733403205872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,2048,0.0124917338291804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,2048,0.017473065853118898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,1536,0.006636799871921539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,1536,0.016697599490483602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,1024,0.0049333333969116214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,1024,0.016124799847602844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,65536,0.24855465888977052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,768,0.004481066763401031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,768,0.015797332922617594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,16384,0.0725109338760376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,512,0.004016000032424927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,12288,0.056029868125915525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,512,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,512,0.0063498665889104204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,256,0.0035573333501815797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,256,0.014973866939544677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,128,0.003302400062481562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,128,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,128,0.005234133203824361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,64,0.0030218665798505146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,64,0.014843733112017313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,32,0.0031690667072931922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,32,0.015067733327547708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,65536,0.10735466480255126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,65536,0.10106879870096844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,51200,0.08613653182983398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,65536,0.2203434626261393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,51200,0.08361706733703614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,16384,0.033838931719462076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,51200,0.17949439684549967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,16384,0.038533333937327066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,12288,0.027830400069554645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,12288,0.035207466284434004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,12288,0.0509610652923584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,3072,0.01744640072186788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,10240,0.024486400683720908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,3584,0.01962239940961202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,10240,0.031162667274475097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,8192,0.020145066579182944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,8192,0.02807786663373311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,7168,0.021398399273554483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,7168,0.026213334004084272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,7168,0.031491200129191085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,6144,0.01867413322130839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,6144,0.025148799022038777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,1536,0.010366933544476827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,1024,0.00823040008544922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,5120,0.01609280010064443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,5120,0.023137066761652628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,5120,0.021894399325052896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,4096,0.013700266679128012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,4096,0.020950400829315187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,3584,0.012109866738319397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,768,0.0072618668278058365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,3584,0.01994453271230062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,3584,0.01813546617825826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,3072,0.011083733042081196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,10240,0.043689600626627606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,3072,0.019166932503382365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,2560,0.008498133222262064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,256,0.005650133391221364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,2048,0.007537066439787547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,2048,0.01730453372001648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,16384,0.06235306660334269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,3072,0.015549866358439126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,1536,0.006072533130645752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,1536,0.009890133142471313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,1536,0.016577066977818807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,2560,0.017895466089248656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,1024,0.004655999938646952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,1024,0.0160480002562205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,768,0.004240000247955322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,768,0.015467733144760132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,768,0.0070816000302632645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,8192,0.03423999945322673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,512,0.0038005332152048744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,512,0.015478400389353433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,256,0.003419733295838038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,256,0.01529706617196401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,6144,0.027654399474461872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,128,0.003101866692304611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,128,0.014947199821472168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,1024,0.00804799993832906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,64,0.0030005333324273427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,64,0.014801067113876343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,32,0.003193599979082743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,32,0.015056000153223673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,2560,0.013773866494496665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,2048,0.011560533444086711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,65536,0.08892800013224283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,65536,0.09386773109436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,51200,0.07252266407012939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,51200,0.07401173114776612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,16384,0.027710932493209838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,16384,0.0400000015894572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,16384,0.06703573067982992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,12288,0.0225983997186025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,12288,0.03261546691258748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,4096,0.01991466681162516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,10240,0.026401066780090333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,10240,0.029858134190241498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,8192,0.021261866887410483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,512,0.00633493314186732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,8192,0.026452267169952394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,7168,0.01923946738243103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,8192,0.03645439942677815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,7168,0.025065600872039795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,256,0.00556160012880961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,6144,0.016992000738779704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,6144,0.028952533006668092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,6144,0.024027733008066814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,5120,0.014908799529075622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,5120,0.02103466590245565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,4096,0.012555733323097229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,128,0.005107200145721436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,4096,0.020526933670043945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,3584,0.011325866977373759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,3584,0.019403733809789023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,3584,0.018798933426539103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,3072,0.010328533252080281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,3072,0.019425066312154134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,3072,0.01609599987665812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,2560,0.0073845331867535904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,2560,0.017779199282328288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,2560,0.014313600460688271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,2048,0.006512000163396199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,2048,0.017053866386413576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,2048,0.011869866649309795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,1536,0.005363200108210246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,1536,0.016448000073432924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,65536,0.22438186009724936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,1024,0.0043935999274253845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,1024,0.015820800264676412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,1024,0.00796693315108617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,768,0.004154666761557261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,768,0.01590079963207245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,12288,0.052033066749572754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,512,0.003740799923737844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,512,0.015382400155067444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,512,0.00619946668545405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,256,0.0034133332471052804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,256,0.014913066228230795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,256,0.005419733126958212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,128,0.0030901332696278887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,128,0.014866133530934652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,128,0.005022933085759481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,64,0.0030730667213598887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,10240,0.04636373519897461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,64,0.014833066860834757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,32,0.002993066608905792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,32,0.014663466811180114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,65536,0.06191786527633667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,65536,0.07617493470509848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,51200,0.04998933474222819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,51200,0.1869503974914551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,65536,0.19933652877807617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,7168,0.03264000018437703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,51200,0.06359039942423503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,16384,0.020507733027140297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,16384,0.0321557343006134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,16384,0.05959999958674113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,12288,0.04628586769104004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,12288,0.028037333488464357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,10240,0.01795626680056254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,5120,0.02466986576716105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,10240,0.025804799795150758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,10240,0.04010453224182129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,8192,0.015165866414705912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,8192,0.03253119985262553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,7168,0.013838932911554972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,4096,0.02137706677118937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,7168,0.022747733195622764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,6144,0.0121888001759847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,6144,0.02261013388633728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,6144,0.025425066550572712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,5120,0.011313066879908244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,5120,0.020473599433898926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,1536,0.009928533434867859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,12288,0.01880106727282206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,4096,0.010583466291427613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,4096,0.018075732390085857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,4096,0.01958079934120178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,3584,0.009426132837931315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,3584,0.018833067019780478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,768,0.006982400019963582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,3072,0.008131200075149536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,8192,0.023509333531061806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,3072,0.01803413430849711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,2560,0.007209600011507671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,2560,0.017629865805308023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,2048,0.006479999919732411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,2048,0.01685439944267273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,51200,0.16437759399414062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,1536,0.005479466418425242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,1536,0.016545066237449647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,1536,0.009484799702962239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,1024,0.004590933521588644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,1024,0.016195199886957803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,1024,0.007659733295440674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,768,0.004226133227348328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,7168,0.02916053334871928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,768,0.016060800353686015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,512,0.0037503999968369803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,512,0.01527466674645742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,512,0.006084266801675161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,256,0.0033013333876927694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,5120,0.021262933810551964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,256,0.01504746675491333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,128,0.0030720000465710956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,128,0.004910933474699656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,128,0.014967466394106546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,64,0.002883200099070867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,32,0.002959999938805898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,64,0.014612266421318054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,32,0.014631467064221701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,65536,0.050215466817220056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,65536,0.07019519805908203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,51200,0.04127039909362793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,65536,0.1782858689626058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,51200,0.05842880010604858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,16384,0.01999893387158712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,51200,0.14443200429280598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,16384,0.028985599676767986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,3584,0.016607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,12288,0.01616426706314087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,12288,0.0274944007396698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,12288,0.03952213525772095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,2560,0.01263146698474884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,2048,0.010940800110499065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,10240,0.015603199601173401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,10240,0.025150932868321735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,8192,0.013083733121554055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,8192,0.027832533915837603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,7168,0.01162559986114502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,7168,0.022665599981943764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,7168,0.024280534187952677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,6144,0.012009599804878235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,6144,0.020966400702794395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,5120,0.010567466417948406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,5120,0.019614932934443156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,5120,0.018667733669281004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,4096,0.009110400080680847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,4096,0.018619734048843383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,4096,0.015666133165359496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,3584,0.008332799871762593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,3584,0.018477867046991982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,3072,0.01446613371372223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,3072,0.007512533167997997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,8192,0.023161600033442177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,3072,0.018168532848358156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,3072,0.012519466876983642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,2560,0.0064650664726893115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,2560,0.017272533973058064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,2048,0.005682133138179779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,768,0.006772266825040181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,2048,0.01665173371632894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,1536,0.004972800115744273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,1536,0.016017066438992818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,256,0.005381333331267038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,1024,0.004339199761549632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,1024,0.015991466244061787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,16384,0.049957334995269775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,768,0.0038965334494908653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,768,0.015537066260973611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,768,0.006410666803518932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,10240,0.03380906581878662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,512,0.0035818666219711304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,512,0.01537493367989858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,256,0.0031989333530267083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,256,0.01554026703039805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,6144,0.02169813315073649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,3584,0.014564266800880432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,128,0.002946133414904277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,128,0.014699733257293702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,128,0.004858666658401489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,64,0.002881066749493281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,64,0.01460693379243215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,32,0.002898133297761281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,32,0.014724266529083253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,65536,0.03856213490168254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,65536,0.0636405348777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,65536,0.14269866943359374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,2560,0.011352533102035522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,51200,0.031196800867716472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,2048,0.00972160001595815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,16384,0.013180800278981528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,51200,0.055087999502817786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,51200,0.11401920318603516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,12288,0.01300266683101654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,1536,0.008725333213806152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,16384,0.027265065908432008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,1024,0.0070613334576288865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,10240,0.011603200435638427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,12288,0.02397866646448771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,10240,0.02245546579360962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,8192,0.011017599701881408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,8192,0.022193066279093423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,8192,0.02138026754061381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,7168,0.010423466563224792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,7168,0.02053119937578837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,6144,0.00974720021088918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,6144,0.020138667027155558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,6144,0.017436800400416057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,5120,0.008796800176302593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,5120,0.020177066326141357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,512,0.005670399963855743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,4096,0.0077461332082748415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,256,0.005241600175698599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,4096,0.012432000041007996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,4096,0.018894932667414346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,3584,0.007245866457621257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,3584,0.018220800161361694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,3584,0.011358933647473653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,3072,0.006818133095900218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,3072,0.018644267320632936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,2560,0.0062282666563987735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,2560,0.017142399152119955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,2048,0.005560533205668131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,2048,0.01665600041548411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,2048,0.008062933385372163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,1536,0.00484799991051356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,1536,0.016249600052833556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,1536,0.007423999905586243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,1024,0.0042250668009122215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,1024,0.01551466683546702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,1024,0.00622080018122991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,768,0.0038399999340375268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,768,0.01548693378766378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,768,0.005731200178464254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,512,0.003554133325815201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,512,0.015280000368754067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,512,0.0054293334484100345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,256,0.0031189332405726117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,256,0.014946132898330688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,12288,0.030551467339197797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,10240,0.02611306707064311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,16384,0.040370134512583415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,128,0.004881066580613455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,128,0.014669866363207499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,64,0.00276799996693929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,7168,0.0195743997891744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,32,0.002867199977238973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,65536,0.028937600056330365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,32,0.014686933159828186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,65536,0.05830826759338379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,65536,0.10264000097910564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,51200,0.02381226619084676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,51200,0.04599786599477132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,5120,0.014829867084821067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,16384,0.00956053336461385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,16384,0.023641600211461385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,12288,0.00942080020904541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,12288,0.0210698664188385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,128,0.002889599899450938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,3072,0.009843200445175171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,10240,0.008762666583061218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,2560,0.009268266956011455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,10240,0.020713599522908528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,8192,0.007410133381684621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,64,0.01474666694800059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,8192,0.020127999782562255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,8192,0.015386666854222616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,7168,0.006841599941253662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,6144,0.00655680000782013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,7168,0.013884799679120383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,7168,0.020382932821909585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,5120,0.006852266689141591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,6144,0.019637332359949747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,5120,0.011384532848993937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,5120,0.01952000061670939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,4096,0.006844800213972728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,51200,0.08107733726501465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,4096,0.018396800756454466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,3584,0.006678399940331777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,3584,0.017909334103266398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,3584,0.00976746678352356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,3072,0.0063701331615448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,3072,0.017790933450063072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,3072,0.008614400029182434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,2560,0.006116266548633576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,2560,0.01698346734046936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,256,0.005042133231957754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,2048,0.005433600147565206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,2048,0.016721065839131674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,2048,0.007267199953397115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,1536,0.004856533308823904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,1536,0.01599360009034475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,1536,0.006939733525117238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,1024,0.004125866790612539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,6144,0.012513066331545511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,1024,0.015783466895421348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,1024,0.006074666480223338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,768,0.003786666691303253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,768,0.005562666555245717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,512,0.0034645333886146545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,512,0.015225600202878317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,16384,0.027923200527826948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,256,0.003159466634194056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,256,0.014685866236686707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,256,0.004916266600290934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,128,0.00288426677385966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,12288,0.021346133947372437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,128,0.004622933268547058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,64,0.002812800059715907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,64,0.0145578662554423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,32,0.0027978666126728057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,32,0.014577066898345948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,65536,0.016361600160598753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,10240,0.0180074671904246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,65536,0.05077013174692789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,51200,0.01237440009911855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,51200,0.04092160065968831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,768,0.015460266669591268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,51200,0.07792853514353434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,16384,0.00827946662902832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,16384,0.02627413272857666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,16384,0.022498132785161336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,12288,0.007779199878374736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,12288,0.0216213325659434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,12288,0.02087999979654948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,10240,0.0068917334079742435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,10240,0.02139520049095154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,10240,0.018151466051737467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,8192,0.00668693333864212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,8192,0.015319466590881348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,8192,0.019436800479888917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,7168,0.006405333181222279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,7168,0.013871999581654868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,6144,0.0062047998110453285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,7168,0.01990506649017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,4096,0.010066133737564088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,128,0.015081600348154704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,5120,0.006628266473611195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,6144,0.012471466263135274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,6144,0.019800533850987755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,5120,0.019351466496785482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,4096,0.0061471998691558834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,5120,0.011319466431935628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,3584,0.006618666648864746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,4096,0.01000426709651947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,3584,0.009621333082516987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,3072,0.006270933151245117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,3584,0.0182805339495341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,3072,0.00844266712665558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,2560,0.006053333481152853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,3072,0.01765973369280497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,2560,0.017091200749079386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,2560,0.008010666569073994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,2048,0.01684160033861796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,2048,0.007150933146476746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,1536,0.006906666855017345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,1536,0.016042666633923848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,2560,0.008101333181063335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,1024,0.015711999932924905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,768,0.0036608000596364343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,1024,0.005995733539263407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,768,0.01541759967803955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,512,0.0033610666791598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,512,0.015092266599337259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,512,0.005142400165398916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,256,0.0030261332790056865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,256,0.01498240033785502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,512,0.005236266553401947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,128,0.002811733384927114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,128,0.01458133359750112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,128,0.004769066472848257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,64,0.0027232001225153605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,4096,0.018433066209157307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,32,0.0028160000840822858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,64,0.014436266819636025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,2048,0.005333333214124044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,1536,0.0046965335806210835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,32,0.014521599809328715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,51200,0.010427733262379963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,1024,0.0040778666734695435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,65536,0.04721173445383708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,16384,0.006750933329264323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,12288,0.0065077334642410275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,51200,0.03756693204243978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,16384,0.022652800877889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,10240,0.00653546651204427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,10240,0.02053119937578837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,12288,0.020667733748753865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,7168,0.00626453310251236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,8192,0.019394133488337198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,6144,0.006020266811052958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,7168,0.019883733987808228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,5120,0.0064074665307998655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,6144,0.019382399320602418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,65536,0.09616320133209229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,4096,0.006061866879463196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,5120,0.01946453253428141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,3584,0.0064298664530118305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,4096,0.018439465761184694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,3072,0.006192000210285186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,3584,0.01790293256441752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,2560,0.005955199897289276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,3072,0.017628800868988038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,2048,0.005287466446558634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,2560,0.01718613306681315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,1536,0.004681600133577982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,1024,0.00402453343073527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,2048,0.01632533371448517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,1536,0.015758933623631795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,768,0.003638399889071783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,1024,0.015203199783960977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,512,0.0033600000043710076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,256,0.0049792001644770306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,768,0.01511253317197164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,256,0.002993066608905792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,128,0.0027722666660944624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,512,0.014921599626541137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,64,0.002703999976317088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,256,0.015246933698654175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,128,0.014601600170135499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,32,0.002700799951950709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,65536,0.010137599706649781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,32,0.01453439990679423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,51200,0.00904319981733958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,65536,0.04599999984105428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,16384,0.006276266773541768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,51200,0.03529493411382039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,12288,0.006385066608587901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,16384,0.021793067455291748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,10240,0.006579199930032094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,65536,0.011095466216405232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,8192,0.006206933160622915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,10240,0.021079466740290324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,7168,0.006146133442719777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,8192,0.020141865809758505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,6144,0.005946666498978933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,7168,0.019678932428359986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,5120,0.0063498665889104204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,6144,0.019108267625172932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,5120,0.019262933731079103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,4096,0.018806399901707967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,3584,0.006378666559855144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,8192,0.006461866696675618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,3072,0.006074666480223338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,3584,0.01793280045191447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,2560,0.006055466830730438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,3072,0.017909334103266398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,2048,0.005308799942334493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,2560,0.016936532656351724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,1536,0.004633600016434988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,2048,0.016707199811935424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,1024,0.003984000037113826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,1536,0.01592853367328644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,1024,0.015643733739852905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,512,0.0033632000287373864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,768,0.015310933192571005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,256,0.0029696000119050344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,256,0.014899200201034546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,128,0.0027978666126728057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,128,0.0144896000623703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,64,0.0026709333062171934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,768,0.00553173323472341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,64,0.014411733547846476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,32,0.0026389333109060925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,64,0.014532267053922018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,32,0.014477866888046264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,12288,0.020595200856526694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,4096,0.0059445331494013464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,768,0.0036906667053699495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,512,0.014949333667755128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,16384,0.41170558929443357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,16384,0.767474110921224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,12288,0.5864469528198242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,12288,0.31603412628173827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,16384,0.7301813125610351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,10240,0.4925248146057129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,12288,0.5750837326049805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,10240,0.2669013341267904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,51200,1.2302965799967447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,10240,0.5177952130635579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,8192,0.3978944142659505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,8192,0.21892800331115722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,7168,0.1935402711232503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,7168,0.3997088114420573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,51200,2.3975306193033856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,6144,0.16935253143310547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,7168,0.39557011922200525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,51200,2.3496116638183593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,6144,0.33717225392659506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,5120,0.14412693977355956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,5120,0.28838186264038085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,5120,0.27350826263427735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,4096,0.12605120340983073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,4096,0.23475200335184732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,4096,0.21435413360595704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,3584,0.1086400032043457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,3584,0.20466559727986655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,3072,0.09643627007802327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,3072,0.17494826316833495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,3072,0.16296000480651857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,2560,0.08446079889933268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,2560,0.14870932896931965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,2048,0.10712746779123943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,2560,0.1358527978261312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,2048,0.07198826471964517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,1536,0.0831775983174642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,2048,0.112335999806722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,6144,0.29877548217773436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,1536,0.0642794648806254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,1024,0.06115946769714355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,768,0.046470399697621664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,1024,0.05016746520996094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,768,0.042423466841379806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,512,0.03408000071843465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,1024,0.061035732428232826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,256,0.01998293399810791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,512,0.03528639872868856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,256,0.025462400913238526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,512,0.032280532519022624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,128,0.01209279994169871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,64,0.009571199615796406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,128,0.025370667378107708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,65536,32,0.00900266667207082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,64,0.021677867571512858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,32,0.020992000897725425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,8192,0.45070934295654297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,1536,0.0838538646697998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,65536,128,0.022012799978256226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,3584,0.18705387115478517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,65536,1.2470527648925782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,768,0.04877973397572835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,65536,2.4161760965983072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,65536,256,0.02940160036087036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,16384,0.6256917317708333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,51200,0.9892800013224283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,51200,1.9128693898518878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,16384,0.32980159123738606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,12288,0.47415679295857743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,10240,0.21533759435017905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,65536,2.3477195739746093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,10240,0.4533418655395508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,16384,0.6399189631144206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,10240,0.427350393931071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,8192,0.3084266662597656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,51200,1.8497151692708331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,7168,0.15720213254292806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,8192,0.19523413976033527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,7168,0.28390188217163087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,8192,0.3736138661702474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,12288,0.2548415978749593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,5120,0.19974826176961263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,6144,0.13806079228719076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,5120,0.12460052967071533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,7168,0.330242125193278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,6144,0.273961607615153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,6144,0.28027518590291345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,4096,0.16285227139790853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,4096,0.1008522669474284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,3584,0.14617172876993817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,5120,0.22464426358540854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,3072,0.12370879650115967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,3584,0.09175360202789307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,3072,0.08127466837565103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,2560,0.10462079842885334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,2560,0.07133653163909912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,2560,0.11532266934712727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,2048,0.08568853537241618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,2048,0.06519253253936767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,1536,0.06749227046966552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,1024,0.04766720136006673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,1536,0.05181866486867269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,1024,0.041605333487192794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,1024,0.051609599590301515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,768,0.03853760162989299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,768,0.03539733489354451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,12288,0.4852277437845866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,512,0.03069546620051066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,512,0.030028800169626873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,512,0.02779200077056885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,256,0.01715839902559916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,256,0.024149332443873087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,256,0.019797333081563315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,128,0.01130560040473938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,128,0.02172799905141195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,128,0.016454399625460307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,64,0.009150933225949604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,64,0.020907733837763467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,51200,32,0.008633599678675333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,51200,32,0.02081600030263265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,3584,0.15097494125366212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,4096,0.17861119906107586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,3072,0.13243412971496582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,2048,0.09167893727620444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,65536,0.7908671696980794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,1536,0.06771519978841146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,65536,0.41455361048380535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,51200,768,0.03941973447799683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,16384,0.19231146176656086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,51200,0.5865877151489258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,51200,0.32967252731323243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,16384,0.1233728011449178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,12288,0.14702399571736652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,10240,0.1232757329940796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,10240,0.08110613028208415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,12288,0.10582719643910725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,8192,0.09949546655019124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,8192,0.06893440087636313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,51200,0.5837557474772136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,7168,0.08736106554667154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,8192,0.1146997372309367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,7168,0.06943146387736002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,7168,0.1006656010945638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,6144,0.07710613409678141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,6144,0.05646506547927856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,5120,0.06509973208109537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,6144,0.08735253016153971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,4096,0.05304960012435913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,5120,0.04996480147043864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,4096,0.04297279914220174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,3584,0.047169065475463866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,3584,0.03996266523996989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,3584,0.05206613143285116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,3072,0.041414399941762284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,3072,0.03669439951578776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,3072,0.04548906485239665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,65536,0.7317696253458659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,2560,0.035488001505533856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,2560,0.03320320049921672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,2048,0.02945706645647685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,2560,0.038610132535298665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,2048,0.029471999406814574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,1536,0.023473066091537476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,1536,0.025513599316279095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,1536,0.025015467405319215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,1024,0.016500266393025716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,1024,0.019402666886647543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,1024,0.0220960001150767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,768,0.012995200355847678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,768,0.02049493392308553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,768,0.0160778671503067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,512,0.009703466296195984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,512,0.019040000438690186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,256,0.006832000116507213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,256,0.016692266861597697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,256,0.0107914666334788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,128,0.004737066725889841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,128,0.01627626617749532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,128,0.009841066598892213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,64,0.004373333354791006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,64,0.016787199179331462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,16384,32,0.004824533561865489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,16384,32,0.016744534174601235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,16384,0.1991114616394043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,65536,0.5852223714192708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,10240,0.13184320131937663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,65536,0.3392437299092611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,5120,0.07351893583933512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,51200,0.4637397448221843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,4096,0.05921279986699422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,51200,0.2736895879109701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,65536,0.6503391901652018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,16384,0.09373119672139486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,16384,0.16804906527201335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,16384,0.17321173350016278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,12288,0.12202560106913249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,12288,0.07591679890950521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,10240,0.09312533537546794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,12288,0.12918506463368734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,10240,0.11381440162658692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,10240,0.06574613253275553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,8192,0.07469759782155355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,51200,0.5070026715596516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,8192,0.05578986803690592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,7168,0.06615893443425497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,2048,0.033089067538579306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,7168,0.05113813479741415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,6144,0.058373332023620605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,6144,0.04699840148289998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,5120,0.049184000492095946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,7168,0.09010132948557535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,6144,0.07874879837036133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,5120,0.041672531763712564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,512,0.013511466979980468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,4096,0.04041706720987956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,5120,0.06370559930801392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,4096,0.036635732650756835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,3584,0.03403306802113851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,3072,0.0318997323513031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,3584,0.0452181339263916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,3072,0.03133119940757752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,2560,0.027471999327341717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,3072,0.04021333456039429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,2048,0.022766933838526408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,2560,0.0283135990301768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,2048,0.025512532393137617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,1536,0.018001067638397216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,1536,0.023031467199325563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,1536,0.02213866710662842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,1024,0.01260586678981781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,1024,0.020787199338277183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,3584,0.035916801293691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,768,0.010443733135859171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,768,0.019540266195933024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,768,0.01447466711203257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,2560,0.034543999036153156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,512,0.008313600222269695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,512,0.016936532656351724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,2048,0.029037866989771528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,256,0.00489386667807897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,256,0.016411733627319337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,128,0.0042346666256586705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,128,0.015737600127855935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,128,0.007909333209196727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,64,0.0038912000755469004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,64,0.016032000382741295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,12288,32,0.004147200038035711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,12288,32,0.016115199526151022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,8192,0.10140906969706218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,65536,0.5048480033874512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,16384,12288,0.1516640027364095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,4096,0.05223679939905802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,65536,0.2976127942403158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,51200,0.24031467437744142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,51200,0.39863999684651696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,16384,0.14347947438557943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,1024,0.017314134041468303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,16384,0.08306986490885417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,16384,0.16850345929463703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,12288,0.0955456018447876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,12288,0.0672106663386027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,12288,0.12344533602396648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,10240,0.07917653719584147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,10240,0.05845226844151815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,10240,0.1091765324274699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,8192,0.06409706672032675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,8192,0.05048853158950806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,7168,0.05714986721674601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,7168,0.04643626610438029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,6144,0.05061759948730469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,7168,0.0852288007736206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,6144,0.04265600045522054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,5120,0.04278186559677124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,6144,0.07302506764729819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,5120,0.03781333367029826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,4096,0.03487679958343506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,5120,0.06133333444595337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,4096,0.03401493231455485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,3584,0.03137600024541219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,3584,0.031566933790842695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,512,0.011610666910807293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,3584,0.04320319890975952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,3072,0.027815467119216918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,3072,0.037265066305796304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,2560,0.024166399240493776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,3072,0.02898026704788208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,12288,256,0.008703999718030294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,2560,0.02645653287569682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,2048,0.020039467016855876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,1536,0.015657599767049155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,2048,0.026778666178385417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,2048,0.024282666047414143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,1536,0.02217493255933126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,1024,0.011447466413180033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,1024,0.020577067136764528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,1024,0.015508266290028891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,768,0.009682133793830872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,768,0.01333440045515696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,768,0.01930560072263082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,512,0.007947733501593272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,512,0.016607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,256,0.004981333514054617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,512,0.010502400000890096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,256,0.01629866659641266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,128,0.004267733295758565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,256,0.00783679982026418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,128,0.015873066584269204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,128,0.007049599786599477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,64,0.0038346665600935614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,64,0.015890133380889893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,10240,32,0.003970133264859518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,10240,32,0.016177067160606386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,65536,0.3895050684611002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,65536,0.5965525309244792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,65536,0.23943146069844565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,51200,0.4874133427937825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,8192,0.09680000146230062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,51200,0.30985708236694337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,65536,0.37382825215657556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,51200,0.19555412928263347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,16384,0.11450133323669434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,16384,0.06968106428782145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,4096,0.051330133279164636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,16384,0.10551573435465496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,12288,0.05695786476135254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,12288,0.07520960172017416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,10240,0.06455146471659343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,10240,0.05034133195877075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,51200,0.30375146865844727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,10240,0.06905600229899088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,2560,0.03271573384602865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,8192,0.05175786813100179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,8192,0.04361813465754191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,7168,0.04543360074361165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,6144,0.040454399585723874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,8192,0.057163735230763756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,7168,0.049356798330942794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,10240,1536,0.020603734254837035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,6144,0.03627626498540242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,5120,0.034354134400685625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,6144,0.04437119960784912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,5120,0.033181866010030106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,4096,0.028138667345046997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,5120,0.037724800904591876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,3584,0.02523626685142517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,4096,0.03142720063527425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,4096,0.029332266251246138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,3584,0.02765013376871745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,3072,0.022501333554585775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,3584,0.027211733659108478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,2560,0.01942080060640971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,3072,0.024988800287246704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,2560,0.021702400843302407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,2048,0.015910399953524272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,1536,0.012307199835777282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,2048,0.018628267447153728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,2048,0.02220266660054525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,1024,0.009241599837938945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,1536,0.020528000593185425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,1024,0.011949867010116577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,1024,0.018769067525863648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,768,0.007968000074227651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,768,0.017529600858688356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,512,0.006293333570162455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,512,0.016787199179331462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,256,0.004642133414745331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,256,0.01585813363393148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,256,0.0075445334116617845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,128,0.0038624001046021783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,128,0.015475199619928996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,7168,0.039953064918518064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,64,0.0036501333117485045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,64,0.015874133507410685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,8192,32,0.004101333270470301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,3072,0.02521066665649414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,32,0.01598186691602071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,8192,2560,0.0237119992574056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,65536,0.21623892784118653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,65536,0.36974719365437825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,65536,0.35878187815348306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,51200,0.29584426879882814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,51200,0.175164794921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,16384,0.1046453317006429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,16384,0.06413760185241699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,12288,0.07943893273671468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,12288,0.06670293013254801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,12288,0.05297600030899048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,12288,0.07340266704559326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,10240,0.05613760153452555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,10240,0.045857067902882895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,10240,0.06425066788991293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,8192,0.04531306823094686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,8192,0.04107946554819743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,7168,0.040394667784372965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,7168,0.037249068419138595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,1536,0.015192533532778421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,6144,0.0359658678372701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,768,0.010641066233317058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,6144,0.040516265233357746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,5120,0.030434133609135945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,512,0.008777599533398945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,5120,0.03610560099283854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,5120,0.0309663991133372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,4096,0.02512960036595662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,4096,0.027189334233601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,3584,0.022581332921981813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,8192,128,0.006910933554172516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,3584,0.02560639977455139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,3584,0.025283199548721314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,3072,0.02016426722208659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,3072,0.0240064005057017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,3072,0.022510933876037597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,2560,0.01716266671816508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,2560,0.02285226583480835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,2560,0.01945599913597107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,2048,0.014010666807492574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,6144,0.03429866631825765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,2048,0.021628799041112264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,1536,0.011172266801198323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,1536,0.019848533471425376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,51200,0.28941119511922203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,1024,0.009564800063769023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,1024,0.01722453236579895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,16384,0.09811519781748454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,1024,0.010918399691581726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,768,0.007316266496976216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,768,0.016370133558909098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,512,0.005318399767080943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,512,0.01635840038458506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,512,0.007880533238252004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,256,0.004261333247025808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,256,0.01578986644744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,8192,0.053557332356770834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,7168,0.04795626799265544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,128,0.004159999887148539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,128,0.005971199770768484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,128,0.015592533349990844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,64,0.003753600021203359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,7168,32,0.003857066730658213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,64,0.015485866864522298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,7168,32,0.01567893326282501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,65536,0.30629653930664064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,4096,0.029685332377751665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,65536,0.19815680185953777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,51200,0.153056001663208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,51200,0.24475520451863608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,65536,0.3330090522766113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,16384,0.08254613081614176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,16384,0.09519893328348795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,16384,0.058898135026295984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,12288,0.06573546727498372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,51200,0.26634880701700847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,12288,0.04864533344904582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,10240,0.05667200088500977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,12288,0.06884160041809081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,8192,0.039985068639119464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,10240,0.04296533266703288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,8192,0.03724693457285563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,10240,0.06040106614430746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,7168,0.03608640034993489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,8192,0.05085226694742838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,7168,0.03466026782989502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,2048,0.01715946594874064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,6144,0.031854933500289916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,6144,0.03204373319943746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,6144,0.039486932754516604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,5120,0.02728533347447713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,5120,0.028893866141637164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,4096,0.02220373352368673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,1536,0.013730133573214212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,5120,0.03395093282063802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,4096,0.025718400875727337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,4096,0.02757013241449992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,3584,0.020142932732899986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,3072,0.017805866400400796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,3584,0.024266666173934935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,3072,0.02294720013936361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,2560,0.01502293348312378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,2560,0.021579732497533165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,2048,0.012289067109425861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,2048,0.016668800512949625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,2048,0.02067520022392273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,1536,0.010228266318639118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,1536,0.013510400056838989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,1536,0.019082667430241902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,1024,0.00811839997768402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,1024,0.017642666896184288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,768,0.006738133231798808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,768,0.016504533092180886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,768,0.0093450665473938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,512,0.005150933563709259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,512,0.01622719963391622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,512,0.007959466675917308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,256,0.004197333256403605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,7168,256,0.0067007998625437425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,3584,0.02510400017102559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,256,0.015809067090352378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,256,0.006483200192451477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,128,0.003734400123357773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,128,0.006026666859785716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,128,0.015598932902018229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,64,0.003656533360481262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,64,0.01548479994138082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,6144,32,0.0037759999434153237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,6144,32,0.01570026675860087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,65536,0.16930239995320637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,65536,0.2514069398244222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,51200,0.20000640551249185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,51200,0.13805333773295086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,7168,0.04564053217569987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,16384,0.07405227025349935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,16384,0.056118400891621914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,3072,0.02210986614227295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,2560,0.018686934312184652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,12288,0.0583402673403422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,1024,0.01065066655476888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,12288,0.044494934876759845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,10240,0.04028480052947998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,10240,0.04703573385874431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,6144,768,0.009287466605504353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,8192,0.034305067857106526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,10240,0.055486933390299475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,8192,0.03452906608581543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,8192,0.049303468068440756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,7168,0.03178986708323161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,7168,0.043834666411081954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,6144,0.027985066175460815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,6144,0.02922133406003316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,5120,0.023924267292022704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,5120,0.026401066780090333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,7168,0.03171520034472148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,65536,0.3135423978169759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,4096,0.0197760005791982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,4096,0.02436586618423462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,3584,0.017925333976745606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,3584,0.02342080076535543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,3584,0.023468800385793052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,3072,0.015635200341542563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,3072,0.02211093306541443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,3072,0.020637865861256918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,2560,0.013317333658536276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,2560,0.021202133099238078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,2560,0.018181333939234413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,2048,0.01121493379275004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,51200,0.25784533818562827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,2048,0.019934932390848793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,1536,0.009327999750773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,2048,0.015595733126004537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,1536,0.012440533439318339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,1536,0.018284799655278523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,1024,0.007477333148320516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,1024,0.016711467504501344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,16384,0.09084160327911377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,768,0.0061951999862988796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,768,0.01632213294506073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,512,0.00453653335571289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,512,0.016289066274960837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,12288,0.06763412952423095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,512,0.007341866691907247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,256,0.004067199925581614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,256,0.015459199746449789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,256,0.006146133442719777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,128,0.0034858666360378264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,128,0.01527253290017446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,64,0.003252266595760981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,64,0.015367466211318969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,5120,32,0.0033589333295822145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,5120,32,0.01532906691233317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,6144,0.03826773166656494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,65536,0.20566186904907227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,65536,0.14457173347473146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,5120,0.03139093319574992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,51200,0.1634954611460368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,65536,0.2963007926940918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,51200,0.11833706696828206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,16384,0.04855999946594238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,16384,0.06463040113449096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,12288,0.04567893346150716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,12288,0.04013013442357381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,10240,0.03728640079498291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,12288,0.060148266951243076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,10240,0.03721706469853719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,8192,0.031227733691533404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,8192,0.03184853394826253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,8192,0.043835731347401936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,7168,0.028016000986099243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,128,0.00561706672112147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,7168,0.029576534032821657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,6144,0.024050132433573405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,4096,0.026349866390228273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,6144,0.027340799570083618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,6144,0.034067201614379886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,5120,0.021124267578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,5120,0.02515946626663208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,5120,0.028989867369333906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,4096,0.017663999398549398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,4096,0.023081600666046143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,3584,0.015366400281588236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,3584,0.02220373352368673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,3584,0.021689599752426146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,3072,0.013745066523551942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,3072,0.02108373244603475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,1024,0.009725866715113322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,2560,0.011980799833933513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,5120,768,0.008653866251309712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,2560,0.020517333348592123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,2560,0.017385600010553996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,2048,0.01035520037015279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,2048,0.0190720001856486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,2048,0.01461120049158732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,1536,0.00848640004793803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,1536,0.017463467518488564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,1536,0.012191999951998394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,1024,0.0067007998625437425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,1024,0.016618667046229045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,1024,0.009585066636403402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,768,0.005559466779232025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,768,0.016239999731381734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,51200,0.22467519442240397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,16384,0.08118506272633871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,512,0.004574933151404063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,512,0.015914666652679443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,10240,0.05061333179473877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,256,0.0037439999481042228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,256,0.01530346671740214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,128,0.003385599950949351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,128,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,7168,0.0395466685295105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,64,0.003139200061559677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,64,0.015146666765213012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,4096,32,0.0034186666210492453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,4096,32,0.015203199783960977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,4096,0.023827199141184488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,65536,0.19315199851989745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,65536,0.14149120648701985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,3072,0.01886720061302185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,65536,0.2793343861897787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,51200,0.1537216027577718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,51200,0.1088864008585612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,16384,0.05038933356602987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,16384,0.04324906667073568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,12288,0.0410645325978597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,12288,0.03642239967981974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,12288,0.061273598670959474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,10240,0.03092479904492696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,10240,0.03246506651242574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,8192,0.025090134143829344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,8192,0.028995199998219805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,768,0.00881599982579549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,7168,0.022772266467412313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,7168,0.026341332991917925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,7168,0.041392000516255696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,6144,0.020415999492009482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,512,0.0075552001595497135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,6144,0.025106134017308553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,5120,0.017338667313257852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,5120,0.02350613276163737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,256,0.006419200201829274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,4096,128,0.005909333129723867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,5120,0.02979946732521057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,4096,0.025385600328445435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,4096,0.021592533588409422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,3584,0.012647466858228049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,3584,0.02158400019009908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,3072,0.011333333452542622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,3072,0.020552533864974975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,3072,0.01907093326250712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,2560,0.010019200046857198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,2560,0.020104533433914183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,4096,0.013931733369827271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,2048,0.008727467060089112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,51200,0.22778026262919107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,16384,0.0840554634730021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,2048,0.014263466993967692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,2048,0.01879146695137024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,1536,0.011352533102035522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,1536,0.017270400126775106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,1024,0.005949866771697998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,1024,0.01664746701717377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,768,0.005129600067933401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,768,0.016249600052833556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,768,0.008202666540940602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,512,0.004532266656557719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,512,0.01565439999103546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,10240,0.05128106673558554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,8192,0.04515626827875773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,256,0.0038954667747020722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,256,0.005938133100668589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,256,0.015261866648991904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,128,0.0033557333052158357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,128,0.014941866199175516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,128,0.005533866584300995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,64,0.0031871999303499854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,32,0.0032405334214369455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,64,0.015122133493423461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3584,32,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,65536,0.15801386833190917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,6144,0.0366154670715332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,65536,0.12176746527353924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3584,1536,0.007946666578451793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,51200,0.12713706493377686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,1024,0.009066667159398396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,16384,0.04633920192718506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,16384,0.04111573298772176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,51200,0.22537600199381508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,12288,0.0364959994951884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,3584,0.021569067239761354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,12288,0.034856534004211424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,10240,0.031890134016672775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,10240,0.031031467517217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,2560,0.016754132509231568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,8192,0.02286613384882609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,8192,0.026709334055582686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,8192,0.04415573279062907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,7168,0.02081386645634969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,7168,0.025967999299367266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,51200,0.10057813326517742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,6144,0.01858666737874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,6144,0.02385279933611552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,6144,0.03452906608581543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,5120,0.01541759967803955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,5120,0.022870399554570518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3584,512,0.00697813332080841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,4096,0.012591999769210816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,4096,0.020872533321380615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,4096,0.02482453385988871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,3584,0.011661866307258606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,3584,0.020503467321395873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,3584,0.021630932887395225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,3072,0.01055999994277954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,3072,0.01954453388849894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,16384,0.08494293689727783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,2560,0.009457066655158997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,2560,0.018681599696477254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,12288,0.06087786753972372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,2560,0.01665066679318746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,2048,0.008133333424727123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,2048,0.014052266875902811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,2048,0.018228266636530557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,1536,0.007055999835332234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,10240,0.050560001532236734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,1536,0.016773333152135216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,1536,0.011001599828402202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,1024,0.005298133194446564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,768,0.004670933385690053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,1024,0.016356266538302102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,768,0.016683733463287352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,512,0.004042666653792063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,512,0.015999999642372132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,512,0.006763733426729838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,256,0.003835733234882355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,256,0.005832533538341522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,256,0.01525759994983673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,128,0.0034346667428811393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,7168,0.039698131879170734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,128,0.005220266679922739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,64,0.0032042667269706728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,64,0.015209600329399109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,3072,32,0.0033269333342711128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,32,0.015156267086664834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,65536,0.13508799870808919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,65536,0.11096426645914714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,65536,0.29076480865478516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,51200,0.10835200150807697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,65536,0.2670079867045084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,51200,0.08666133085886638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,5120,0.02948906620343526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,16384,0.040192000071207684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,12288,0.03132373293240865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,1024,0.008752000331878663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,12288,0.03175786733627319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,12288,0.06206933259963989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,10240,0.02765866716702779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,10240,0.028116265932718914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,10240,0.04896106719970703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,8192,0.020526933670043945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,8192,0.025595732529958087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,8192,0.04328426520029704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,7168,0.01882773240407308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,7168,0.024643200635910033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,7168,0.03833386500676473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,6144,0.016229333480199178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,6144,0.023201066255569457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,6144,0.03418240149815877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,5120,0.0139765332142512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,3072,0.01882986625035604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,4096,0.011634133259455363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,5120,0.028757333755493164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,4096,0.02069759964942932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,3584,0.010871466994285584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,3584,0.02074133356412252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,16384,0.03895253340403239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,3072,0.009688533345858256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,3072,0.01948266625404358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,3072,0.018396800756454466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,2560,0.008806399504343669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,2560,0.018574933211008705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,2560,0.01593386630217234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,2048,0.007604266703128815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,2048,0.01729173262914022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,2048,0.013431466619173684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,3072,128,0.015130666891733804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,1536,0.006628266473611195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,3072,768,0.008053333560625712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,1536,0.010744532942771912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,1536,0.017121066649754844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,1024,0.0050357331832249965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,1024,0.016042666633923848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,768,0.004515199859937032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,768,0.01599679986635844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,5120,0.021875200668970744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,512,0.004086400071779886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,512,0.016005333264668783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,51200,0.2210773309071859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,16384,0.08026986916859945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,256,0.003583999971548716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,256,0.0151829332113266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,128,0.003252266595760981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,128,0.015015467007954916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,128,0.0052042668064435325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,64,0.0030303999781608583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,64,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2560,32,0.0030933332939942675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2560,32,0.01495680014292399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,65536,0.11127466360727947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,65536,0.09388906955718994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,4096,0.023894399404525757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,3584,0.02121280034383138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,51200,0.08981653054555258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,51200,0.07759466965993246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,65536,0.24581759770711265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,16384,0.0330570658047994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,16384,0.03625919818878174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,12288,0.026293333371480303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,12288,0.030701865752538044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,12288,0.052349865436553955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,10240,0.022997333606084188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,10240,0.02755733331044515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,1024,0.00865280032157898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,8192,0.019368533293406168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,768,0.008014933268229166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,8192,0.024804266293843587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,8192,0.038072534402211505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,7168,0.017375999689102174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,7168,0.03374079863230388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,6144,0.01699519952138265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,7168,0.02389226754506429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,5120,0.01295253336429596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,512,0.006687999765078227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,5120,0.021719467639923096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,5120,0.026035199562708538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,4096,0.02062826752662659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,4096,0.021545600891113282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,3584,0.011174399654070537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,3584,0.019242666165033975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,3584,0.018615466356277467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,3072,0.010327466328938802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,3072,0.018618667125701906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2560,256,0.005756799876689911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,2560,0.008635733524958293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,2560,0.017442133029301962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,2560,0.01458133359750112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,2048,0.008051200211048127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,2048,0.017352533340454102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,6144,0.022805333137512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,1536,0.006045866509278615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,1536,0.01723519961039225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,1536,0.010191999872525533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,1024,0.004878933231035868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,1024,0.016170666615168253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,16384,0.07118933200836182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,768,0.004337066908677419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,51200,0.20104533831278482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,768,0.015425067146619162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,512,0.0038346665600935614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,512,0.015267200271288552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,512,0.00680320014556249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,256,0.003402666747570038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,256,0.014822399616241455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,10240,0.04335999886194865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,256,0.005723733206590017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,128,0.003222399950027466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,128,0.014699733257293702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,128,0.005301333467165629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,64,0.003047466774781545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,32,0.0031541332602500914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,64,0.015031466881434122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,2048,32,0.014694399634997048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,65536,0.08741439978281657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,6144,0.031067732969919843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,65536,0.0827392021814982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,51200,0.07072106997172037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,51200,0.06800320148468017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,65536,0.24823573430379234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,16384,0.02648319999376933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,51200,0.19776426951090495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,16384,0.03151146570841472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,16384,0.07654826641082764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,12288,0.02111039956410726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,12288,0.029308799902598066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,10240,0.01830079952875773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,12288,0.05647679964701334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,10240,0.026821333169937133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,8192,0.01495680014292399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,10240,0.04774186611175537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,7168,0.016613333423932394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,3072,0.016582399606704712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,7168,0.023543467124303184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,7168,0.0392138679822286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,6144,0.01418880025545756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,6144,0.022182399034500123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,6144,0.032839467128117875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,2048,4096,0.011242666840553283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,5120,0.02139413356781006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,5120,0.02767680088678996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,4096,0.01069760024547577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,4096,0.01935466726620992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,4096,0.02294506629308065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,3584,0.009795199831326802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,3584,0.019223467508951823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,1024,0.008339200417200725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,3584,0.019579732418060304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,3072,0.008923733234405517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,3072,0.01834026575088501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,2560,0.008087466657161712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,2560,0.018088533480962118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,8192,0.02436586618423462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,8192,0.041832534472147624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,2048,0.006696533163388569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,2048,0.01308799982070923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,1536,0.005726933479309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,1536,0.01684266726175944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,1536,0.010418132940928141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,1024,0.004891733328501383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,768,0.007732266684373219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,1024,0.016189866264661155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,768,0.004530133306980133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,5120,0.01244160036245982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,768,0.015778133273124696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,2048,2048,0.012405332922935487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,512,0.0039893334110577905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,512,0.015516799688339234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,256,0.0034346667428811393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,256,0.0276853342851003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,256,0.005425066749254862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,128,0.0030975999931494398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,128,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,128,0.005120000243186951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,64,0.003005866706371307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,64,0.014888532956441245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1536,32,0.003138133386770884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,32,0.014961066842079162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,65536,0.0598634680112203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1536,2048,0.01717546582221985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,65536,0.06959359645843506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,51200,0.04841386477152507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,65536,0.21873706181844077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,51200,0.057367467880249025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,16384,0.019082667430241902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,51200,0.17542293866475422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,16384,0.027029333511988322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,12288,0.014751999576886495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,16384,0.06633173227310181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,12288,0.02783893346786499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,12288,0.049304533004760745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,10240,0.013382400075594584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,10240,0.02617599964141846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,8192,0.011425066987673442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,8192,0.024026666084925333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,7168,0.012859732906023661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,7168,0.022846933205922446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,3072,0.01757546663284302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,6144,0.013119999567667642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,6144,0.021851734320322672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,2560,0.01474133332570394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,5120,0.011251200238863628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,5120,0.020632533232371013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,5120,0.023692800601323446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,4096,0.010653866330782573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,4096,0.01948266625404358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,768,0.007430399954319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,3584,0.008920533458391826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,1024,0.008353066444396973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,3584,0.01840959986050924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,3072,0.008277333279450735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1536,512,0.006539733211199443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,8192,0.03656320174535115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,2560,0.007268266876538594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,3072,0.015402666727701821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,2560,0.013538133104642233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,2560,0.017488000790278117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,2048,0.00631039987007777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,2048,0.0170741339524587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,1536,0.0054730668663978575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,1536,0.01629973351955414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,1536,0.010061867038408915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,1024,0.00462719996770223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,1024,0.015685333808263143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,1024,0.008316799998283386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,768,0.004101333270470301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,768,0.015271466970443726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,768,0.007563733557860057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,512,0.003705599904060364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,512,0.015099733074506124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,3584,0.01697493394215902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,256,0.003205333401759466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,3072,0.017898666858673095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,256,0.014813866217931113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,256,0.00550186683734258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,128,0.0029706666866938275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,128,0.014670933286348978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,7168,0.032762666543324784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,64,0.002867199977238973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,64,0.014756266276041666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,1024,32,0.00290133332212766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,1024,32,0.014822399616241455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,65536,0.048547200361887616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,65536,0.06320639848709106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,6144,0.027985066175460815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,51200,0.03974293470382691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,51200,0.053617068131764735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,4096,0.01973653237024943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,16384,0.01454080045223236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,16384,0.02837760051091512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,10240,0.04107413291931152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,12288,0.012109866738319397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,12288,0.024573866526285806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,2048,0.011582932869593303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,10240,0.011849600076675414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,10240,0.025126399596532185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,10240,0.034832000732421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,8192,0.01290773351987203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,8192,0.02307200034459432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,512,0.006502399841944377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,7168,0.011884799599647522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,7168,0.02226346731185913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,1024,128,0.005038933455944061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,6144,0.010563199718793232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,6144,0.020437333981196085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,5120,0.013081600268681845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,5120,0.02007466753323873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,5120,0.020089600483576456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,4096,0.010947199662526448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,4096,0.019142399231592812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,65536,0.19396479924519855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,3584,0.009097599983215332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,3584,0.01825493375460307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,3584,0.014920533696810404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,3072,0.008096000055472057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,51200,0.15987733205159504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,3072,0.017853866020838417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,16384,0.054205866654713955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,2560,0.017079466581344606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,12288,0.041756800810496014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,2048,0.00629013329744339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,2048,0.017001599073410034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,2048,0.010098133484522502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,1536,0.005051733553409576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,1536,0.016359466314315795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,8192,0.029602134227752687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,1024,0.004347733159859975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,1024,0.015770666797955833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,1024,0.007413333157698314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,768,0.004012800008058548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,768,0.015480533242225647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,2560,0.00659093310435613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,768,0.00678719977537791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,512,0.003706666578849157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,512,0.015127467115720114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,256,0.003289599965016047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,256,0.014916266997655234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,256,0.005380266904830932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,7168,0.026476800441741943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,128,0.003010133405526479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,128,0.014855466286341348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,64,0.002791466563940048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,64,0.014798933267593383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,768,32,0.002883200099070867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,768,32,0.014808533589045205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,6144,0.02313813368479411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,65536,0.03508479992548625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,65536,0.05644586483637491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,51200,0.028838400046030683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,51200,0.0465770681699117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,51200,0.12062719662984211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,4096,0.01686506668726603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,16384,0.013190399607022604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,16384,0.025206400950749712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,12288,0.010406399766604107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,12288,0.02248959938685099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,3072,0.013353600104649862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,12288,0.03415786822636922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,10240,0.011326932907104492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,10240,0.020911999543507895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,10240,0.02906026641527812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,8192,0.01058666706085205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,2560,0.011513599753379821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,8192,0.022651733954747517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,7168,0.009654399752616883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,8192,0.025345067183176678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,7168,0.02138239940007528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,7168,0.02205866575241089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,6144,0.019342933098475137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,6144,0.020359466473261513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,5120,0.0075434664885203045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,5120,0.019720532496770225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,5120,0.017116800944010416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,4096,0.00831573357184728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,4096,0.01853440006573995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,4096,0.013615999619166055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,3584,0.007719466586907704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,3584,0.017862399419148765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,3584,0.012703999876976013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,1536,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,3072,0.00699946681658427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,3072,0.017694934209187826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,2560,0.0063296000162760425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,2560,0.01730026602745056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,2560,0.010265599687894185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,2048,0.005554133156935374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,2048,0.01684053341547648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,512,0.00606826643149058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,1536,0.004807466765244802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,6144,0.008707200487454731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,1536,0.016200533509254454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,1024,0.004142933338880539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,1024,0.015687466661135355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,768,128,0.004901333153247834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,768,0.003804799914360046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,768,0.015458133816719056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,3072,0.011054933071136475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,512,0.0035082665582497918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,65536,0.14995412826538085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,512,0.015312000115712484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,256,0.003222399950027466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,256,0.014852266510327658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,256,0.005141333242257436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,128,0.0029258665939172106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,128,0.014574933052062988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,16384,0.04603093465169271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,64,0.002868266652027766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,64,0.014898133277893067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,512,32,0.002780800064404805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,512,32,0.014679466684659323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,65536,0.024618667364120484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,65536,0.04923839966456096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,65536,0.10931946436564129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,51200,0.01989013353983561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,51200,0.03886186679204305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,51200,0.08525119622548422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,16384,0.010327466328938802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,16384,0.02365866700808207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,12288,0.008843732873598735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,12288,0.021388800938924153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,12288,0.021938133239746093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,10240,0.010331733028093974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,10240,0.020385066668192543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,10240,0.01869866649309794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,8192,0.008915199836095174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,8192,0.019669334093729653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,2048,0.008897067109743754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,7168,0.008228266735871632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,7168,0.01984106699625651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,7168,0.014467199643452963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,6144,0.007469866673151653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,6144,0.019394133488337198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,6144,0.01307093302408854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,5120,0.006712533533573151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,1024,0.00661653329928716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,5120,0.019219199816385903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,4096,0.006309333443641663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,4096,0.018424532810846963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,768,0.006163200239340464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,4096,0.010174933075904845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,3584,0.0066655998428662615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,3584,0.01804693341255188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,3072,0.00631466656923294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,3072,0.017476266622543334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,3072,0.00860053300857544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,2560,0.006090666850407918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,2560,0.016772266228993735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,2560,0.00804373323917389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,2048,0.005445333321889242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,512,0.005594666798909505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,2048,0.016976000865300496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,2048,0.007187200089295705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,1536,0.004779733220736186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,1536,0.006971733272075653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,128,0.004806399842103322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,1024,0.004109866668780645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,1024,0.01564479966958364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,768,0.003739733248949051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,768,0.015438933173815408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,768,0.005634133517742157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,512,0.0035018667578697203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,512,1536,0.007394133508205414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,512,0.005258666475613912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,512,0.01542080044746399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,256,0.0030986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,256,0.015052800377209982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,256,0.004950400193532308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,128,0.002899199972550074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,128,0.014404267072677612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,64,0.0027029333015282948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,64,0.014654933412869772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,256,32,0.0027072000006834666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,16384,0.02954453428586324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,32,0.014553599556287131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,65536,0.016657066345214844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,65536,0.04325439929962158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,256,1536,0.01611733337243398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,51200,0.01427839994430542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,1024,0.005958400170008342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,65536,0.09893013636271158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,16384,0.007809066772460937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,51200,0.03555519978205363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,8192,0.016196266810099284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,12288,0.007416533430417378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,16384,0.025693867603937787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,12288,0.020578134059906005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,12288,0.020884267489115396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,10240,0.007001600166161854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,10240,0.02001919945081075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,8192,0.006788266698519389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,8192,0.020132267475128175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,5120,0.011614933609962463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,8192,0.014830933014551798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,7168,0.00643093337615331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,7168,0.019428267081578573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,7168,0.013588266571362815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,6144,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,6144,0.012547199924786886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,5120,0.00644053320089976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,3584,0.009494400024414063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,5120,0.0194048007329305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,5120,0.011211733023325603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,4096,0.006017066538333893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,4096,0.009925333658854167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,4096,0.018227199713389076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,3584,0.006499200065930684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,3584,0.01803413430849711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,3584,0.009296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,3072,0.006165333092212677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,3072,0.017614932854970296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,2560,0.006029866635799408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,16384,0.022793600956598915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,2560,0.01685439944267273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,2048,0.0053610667586326596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,2048,0.016582399606704712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,2048,0.0070613334576288865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,1536,0.004670933385690053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,1536,0.015893333156903586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,1536,0.006735999882221222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,1024,0.004086400071779886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,6144,0.006151466568311056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,1024,0.01569919983545939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,768,0.003732266773780187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,1024,0.005892266829808554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,768,0.005622399846712748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,768,0.015244799852371215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,512,0.015278933445612588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,512,0.005209599932034811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,256,0.0030293333033720653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,256,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,128,0.002812800059715907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,128,0.014830933014551798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,51200,0.07670933405558268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,64,0.0026528000831604003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,64,0.014548266927401224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,32,0.0026709333062171934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,256,128,0.004834133386611939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,65536,0.01223466694355011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,10240,0.017607466379801432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,128,512,0.0034111998975276947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,51200,0.01100373367468516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,65536,0.039061331748962404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,16384,0.006877866884072621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,51200,0.033460267384847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,12288,0.00622080018122991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,16384,0.022037333250045775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,10240,0.0065077334642410275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,12288,0.019761067628860474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,10240,0.01999573310216268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,8192,0.0069578667481740315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,7168,0.006362666686375936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,8192,0.019203199942906698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,6144,0.00598826656738917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,7168,0.019641600052515664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,5120,0.00647680014371872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,6144,0.01885120073954264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,4096,0.006052266558011373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,4096,0.018075732390085857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,3584,0.017588265736897788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,3072,0.0060810665289560955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,3072,0.01752426624298096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,2560,0.005974400043487549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,3072,0.008292266726493835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,2560,0.01691840092341105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,2048,0.0053962667783101406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,1536,0.004690133531888326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,2560,0.008081066608428954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,2048,0.016426666577657064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,1024,0.004029866556326548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,768,0.003638399889071783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,1536,0.015801599621772765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,512,0.0033728001018365227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,768,0.015461333592732749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,128,32,0.014548266927401224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,256,0.0029365333418051405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,512,0.015144532918930054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,128,0.0028746667007605235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,256,0.014775466918945313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,64,0.0026389333109060925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,128,0.014404267072677612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,64,0.014574933052062988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,32,0.0026208000878492994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,65536,0.01655359963575999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,32,0.014495999614397685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,51200,0.014163200060526529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,65536,0.03761066595713298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,16384,0.007719466586907704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,51200,0.03540159861246745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,12288,0.006267733375231425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,16384,0.021333332856496176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,10240,0.006321066617965698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,12288,0.020296533902486168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,5120,0.01920213301976522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,8192,0.006636799871921539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,64,3584,0.006420266628265381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,10240,0.019989333550135293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,7168,0.006331733365853627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,8192,0.01926079988479614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,6144,0.005952000121275584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,5120,0.006389333307743073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,7168,0.01925119956334432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,6144,0.019248000780741372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,4096,0.00592853327592214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,256,0.004925866425037384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,5120,0.019151999553044637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,4096,0.018544000387191773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,3072,0.005998933315277099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,3584,0.017784533898035686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,2560,0.006099199752012888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,3072,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,2048,0.0054282665252685545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,2560,0.016742400328318276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,1536,0.004657066861788432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,2048,0.01660053332646688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,1024,0.0039381332695484165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,1536,0.016057599584261575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,768,0.0036949334045251214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,1024,0.015496533115704855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,512,0.003443199892838796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,768,0.015134933590888976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,256,0.003222399950027466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,512,0.015221333503723145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,128,0.0027722666660944624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,256,0.014858667055765787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,64,0.0027488000690937043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,128,0.014548266927401224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,32,0.002737066646416982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,64,1024,0.015501866738001505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,64,0.014502400159835815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,129,32,32,0.01456000010172526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,16384,0.409775988260905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,129,32,3584,0.0063360000650088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,16384,0.7717344284057617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,129,128,128,0.004725333551565806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,16384,0.3767423947652181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,12288,0.28536640803019203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,12288,0.3149034818013509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,12288,0.5850048065185547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,10240,0.26592639287312825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,51200,1.1332266489664713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,10240,0.23908586502075196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,51200,1.2268341064453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,8192,0.3928661346435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,10240,0.49175148010253905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,8192,0.21834452946980795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,7168,0.19304426511128742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,6144,0.29853973388671873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,6144,0.1510815938313802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,6144,0.1701621373494466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,7168,0.19724052747090656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,7168,0.3532160123189291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,5120,0.24996479352315268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,51200,2.3919967651367187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,4096,0.12095999717712402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,5120,0.14437333742777508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,4096,0.1183893362681071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,4096,0.2026048024495443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,3584,0.17842027346293132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,3584,0.1207157293955485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,3072,0.1547221342722575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,3584,0.09505173365275064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,3072,0.09638186295827231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,2560,0.13089173634847004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,3072,0.0856607993443807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,2048,0.07143253485361735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,2048,0.10661439895629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,2560,0.09275093078613281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,1536,0.059146666526794435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,1024,0.06551573276519776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,1024,0.050228265921274814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,768,0.04624319871266683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,8192,0.19483307202657063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,768,0.040133333206176756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,768,0.03190186619758606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,512,0.03411733309427897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,512,0.031991465886433916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,512,0.024627200762430825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,256,0.019730132818222047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,256,0.02515946626663208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,5120,0.12904746532440187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,128,0.01269546647866567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,128,0.02158720095952352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,128,0.016341333587964378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,64,0.0130431999762853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,1536,0.0832373301188151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,64,0.020606933037439983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,32,0.013451733191808066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,32,0.02027626633644104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,1536,0.052889601389567054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,2560,0.07212693691253662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,2048,0.06216959953308106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,1024,0.03637119928995768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,65536,1.2288426717122396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,256,0.018655999501546224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,65536,2.383527374267578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,16384,0.601583989461263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,16384,0.3286879857381185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,51200,0.9703082402547201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,12288,0.2694176038106283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,12288,0.5353365580240885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,51200,1.874062983194987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,10240,0.39673067728678385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,10240,0.21604480743408203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,8192,0.17735999425252277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,8192,0.35772692362467445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,7168,0.2690090815226237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,7168,0.176585594813029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,6144,0.23810879389444986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,6144,0.1390773296356201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,12288,0.23040000597635904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,5120,0.19989546140034992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,65536,1.1676021575927735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,5120,0.11900586287180584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,4096,0.16212693850199383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,4096,0.09915520350138346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,10240,0.22271893819173177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,16384,0.30357653299967446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,6144,0.12190506458282471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,3584,0.16237012545267743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,51200,0.9079455693562826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,3584,0.09282346566518149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,3072,0.12449599901835125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,3072,0.08962453206380208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,2560,0.10509333610534669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,2560,0.07758933703104655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,7168,0.13979093233744305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,2048,0.08632213274637858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,8192,0.15757226943969727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,2048,0.06227733294169108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,1536,0.07506346702575684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,1536,0.05162133375803629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,1024,0.04750399986902873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,1024,0.04151893456776937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,3072,0.06703680356343587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,4096,0.08531200091044108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,768,0.03832319974899292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,768,0.036218667030334474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,512,0.02853333353996277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,512,0.030210133393605548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,1536,0.03884373505910237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,256,0.017544533809026083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,256,0.024954666694005333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,5120,0.11585813363393146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,128,0.012405332922935487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,2560,0.05815999905268351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,128,0.023399466276168825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,64,0.010226133465766906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,64,0.022796799739201866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,51200,32,0.010602666934331258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,51200,32,0.022986666361490885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,1024,0.030158933003743487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,3584,0.0757258653640747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,2048,0.04765119949976603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,65536,0.4172959963480632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,65536,0.7828800201416015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,65536,0.38001600901285804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,16384,0.19165652592976887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,51200,0.58941650390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,51200,0.33117758433024086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,51200,0.30659732818603513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,16384,0.13442346254984539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,16384,0.11935573418935139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,12288,0.1558026631673177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,12288,0.10484906832377117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,10240,0.12273706595102947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,10240,0.08080213069915772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,10240,0.06758933067321778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,8192,0.09879679679870605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,8192,0.06828800042470297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,7168,0.08792533079783121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,7168,0.06204479932785034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,7168,0.05021653175354004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,6144,0.0769045352935791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,6144,0.056005334854125975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,6144,0.04455893437067668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,5120,0.06450026830037435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,5120,0.04986240069071452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,5120,0.03887573480606079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,4096,0.0529695987701416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,4096,0.04296213388442993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,4096,0.032671999931335446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,3584,0.04700800180435181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,3584,0.04010773499806722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,3584,0.029628799359003706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,8192,0.0560533324877421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,3072,0.041300264994303386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,3072,0.03678719997406006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,3072,0.026730666557947796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,2560,0.0353770653406779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,2560,0.03342506488164266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,2560,0.0236298660437266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,2048,0.02928000092506409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,2048,0.019989333550135293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,2048,0.0294378658135732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,256,0.015060266852378846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,1536,0.023449599742889404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,1536,0.025220266977945965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,1024,0.022078933318456014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,1024,0.012972799936930337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,768,0.012716799974441528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,768,0.020381865898768108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,128,0.013180800278981528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,512,0.009475200374921163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,512,0.01933120091756185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,512,0.010125866532325745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,256,0.006808533271153767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,512,0.019689599672953285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,256,0.016936532656351724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,128,0.0046015997727712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,128,0.01597653329372406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,12288,0.07972373167673746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,128,0.0071381335457166035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,64,0.0043488000830014546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,32,0.004739200075467428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,64,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,32,0.0165802667538325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,1024,0.016410666704177856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,51200,768,0.025276799996693928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,65536,0.3370933214823405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,65536,0.581865628560384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,65536,0.29707520802815757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,51200,0.45990400314331054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,16384,0.14201386769612628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,16384,0.09271466732025146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,51200,0.25205866495768225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,16384,0.09357546965281169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,12288,0.10901119709014892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,12288,0.07428373495737711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,12288,0.06774400075276693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,10240,0.0921727975209554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,10240,0.053529600302378334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,10240,0.0646016001701355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,8192,0.07382293542226157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,8192,0.055257598559061684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,7168,0.06530880133310954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,8192,0.04453013340632121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,7168,0.05081599950790405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,7168,0.039580798149108885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,6144,0.05777920087178549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,6144,0.04588799873987834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,6144,0.03521600166956584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,5120,0.04864960114161174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,5120,0.04123946825663249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,4096,0.03986560106277466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,4096,0.03601919809977214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,1536,0.016343466440836587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,3584,0.03354666630427043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,3072,0.031413332621256514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,3072,0.031099732716878253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,768,0.011741866668065388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,2560,0.02723840077718099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,2560,0.027909332513809205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,2560,0.018598399559656777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,2048,0.022502400477727256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,256,0.00746666689713796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,2048,0.015659733613332113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,2048,0.025068799654642742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,1536,0.01795626680056254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,1536,0.02285760045051575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,1536,0.013035733501116434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,1024,0.012387200196584066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,1024,0.019990400473276774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,768,0.01002346674601237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,5120,0.030836266279220582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,768,0.019006933768590292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,512,0.0077898666262626644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,512,0.016361600160598753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,512,0.007086933155854543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,256,0.004975999891757965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,256,0.01573013365268707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,256,0.0064181332786877945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,128,0.004182399809360504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,51200,0.2454592068990072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,128,0.015293866395950317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,3584,0.03556053241093953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,128,0.006222933530807495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,64,0.0038901334007581077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,32,0.004300799965858459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,64,0.015994667013486227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,32,0.016214399536450704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,4096,0.025860265890757246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,3584,0.023500800132751465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,3072,0.02099306583404541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,65536,0.2962208112080892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,65536,0.49607893625895183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,65536,0.25529600779215494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,51200,0.39566612243652344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,16384,0.14100799560546876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,51200,0.2184768040974935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,16384,0.0816927989323934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,51200,0.19999465942382813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,16384,0.07990293502807617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,12288,0.09314453601837158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,12288,0.06625706752141317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,10240,0.07812373638153076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,10240,0.057417599360148106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,10240,0.046012798945109054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,8192,0.06291093428929648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,8192,0.049788800875345866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,8192,0.038413866360982256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,7168,0.05589439868927002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,7168,0.045363199710845944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,6144,0.04910720189412435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,6144,0.04114666779836019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,1024,0.010965333382288615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,5120,0.04159359931945801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,5120,0.03688106536865234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,4096,0.03421546618143718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,768,0.009741866588592529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,4096,0.03287573258082072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,3584,0.030624000231424968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,3584,0.020518400271733604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,3072,0.027093333005905152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,3072,0.028038400411605834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,2560,0.02317439913749695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,2560,0.025620265801747637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,6144,0.030379732449849445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,2048,0.019172267119089762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,5120,0.026573866605758667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,2048,0.023435733715693154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,4096,0.02238933245340983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,1536,0.021652267376581828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,3584,0.030508800347646074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,1024,0.01071679989496867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,1024,0.019250132640202842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,12288,0.054202667872111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,768,0.010107733805974324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,768,0.018582399686177573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,1536,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,512,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,7168,0.03417173226674398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,1536,0.011827199657758077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,256,0.0046623999873797095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,512,0.006607999900976817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,256,0.01590720017751058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,128,0.003967999915281932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,128,0.015586133797963461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,128,0.005554133156935374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,64,0.0037823999921480812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,3072,0.017960532506306966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,64,0.015889066457748412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,32,0.004261333247025808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,32,0.015924266974131265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,2560,0.015966932972272238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,65536,0.2382805347442627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,65536,0.41023893356323243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,512,0.016846932967503867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,51200,0.1948522726694743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,51200,0.3290805180867513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,2048,0.013691733280817667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,16384,0.10964907010396321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,16384,0.07517866293589273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,12288,0.07460479736328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,1024,0.009895466764767965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,12288,0.056677333513895665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,10240,0.06268266836802164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,10240,0.04973013401031494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,8192,0.05070186853408813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,8192,0.04314986864725749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,768,0.008531199892361958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,7168,0.04506880044937134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,7168,0.03973546822865804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,256,0.0060586666067441305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,6144,0.03986453215281169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,6144,0.03652266661326091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,65536,0.20863466262817382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,5120,0.03386346499125163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,5120,0.032638933261235556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,51200,0.16504426002502443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,4096,0.028548266490300494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,12288,0.04535359938939412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,4096,0.029022934039433797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,16384,0.05841813484827677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,3584,0.02500266631444295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,10240,0.0387829343477885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,3584,0.02685439984003703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,3072,0.02216959993044535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,3072,0.025243733326594037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,8192,0.03218239943186442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,2560,0.018989866971969603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,2560,0.02344533403714498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,7168,0.02892586588859558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,2048,0.01556373337904612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,2048,0.021927465995152794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,6144,0.02584213415781657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,1536,0.012141866485277812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,1536,0.020231467485427857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,1024,0.009186133742332458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,1024,0.018474666277567546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,5120,0.02247040073076884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,768,0.00786133309205373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,768,0.016266666849454246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,4096,0.019351466496785482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,512,0.006412800153096516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,512,0.01635199983914693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,3072,0.015565866231918335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,256,0.00413973331451416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,256,0.015761066476504007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,2048,0.012418133020401002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,128,0.00363520011305809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,3584,0.017718400557835898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,128,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,64,0.0034506666163603462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,64,0.01567893326282501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,32,0.0037952000896135964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,32,0.015800533692042033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,2560,0.014135467012723288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,65536,0.2125930627187093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,65536,0.3675349235534668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,51200,0.28338559468587243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,1024,0.009177600344022114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,51200,0.1721120039621989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,1536,0.011010133226712545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,16384,0.09710720380147299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,16384,0.06377280155817668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,12288,0.07441706657409668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,12288,0.05284693241119385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,768,0.007156266768773396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,512,0.006554666658242543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,256,0.006153599917888641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,10240,0.05617493391036987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,10240,0.046366933981577554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,8192,0.04509119987487793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,8192,0.03945706685384114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,128,0.005838933090368906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,7168,0.040438401699066165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,7168,0.03685226837793986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,6144,0.03590826590855916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,6144,0.03373440106709798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,65536,0.19226773579915363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,5120,0.034652801354726155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,12288,0.04162773291269938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,5120,0.0301962673664093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,4096,0.02467093269030253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,4096,0.026929066578547163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,16384,0.05366506576538086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,3584,0.021975467602411904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,3584,0.02515520056088765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,51200,0.15243733723958333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,3072,0.0220960001150767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,3072,0.0236842672030131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,6144,0.023880533377329507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,2560,0.016721065839131674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,2560,0.022434133291244506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,10240,0.035837864875793456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,2048,0.013662933309872945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,7168,0.026723200082778932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,2048,0.02107200026512146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,1536,0.010872532924016316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,1536,0.019817600647608437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,8192,0.029649066925048827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,1024,0.008403199911117553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,1024,0.017667200167973837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,3072,0.014311466614405313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,768,0.007180800040562947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,768,0.016285866498947144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,4096,0.017588265736897788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,512,0.005096533397833506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,512,0.015943466623624166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,5120,0.020829866329828896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,256,0.003940266619126002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,256,0.015287466843922935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,3584,0.015688533584276833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,128,0.0035616000493367515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,128,0.015197867155075073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,2560,0.013035733501116434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,1536,0.009890133142471313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,64,0.0033557333052158357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,32,0.003751466671625773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,64,0.015460266669591268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,32,0.01530880033969879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,1024,0.007999999821186066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,65536,0.20570774078369142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,65536,0.3230954806009928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,2048,0.011411199967066448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,51200,0.2532575925191244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,51200,0.16332906087239582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,16384,0.08200853665669759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,16384,0.058164266745249424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,256,0.005807999769846598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,128,0.005612800021966299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,512,0.006243200103441874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,12288,0.056932266553243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,10240,0.04838720162709554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,12288,0.04766933520634969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,768,0.006666666766007741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,10240,0.041819731394449874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,8192,0.039043200016021726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,8192,0.036962131659189865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,7168,0.034891732533772785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,7168,0.03371626536051432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,6144,0.03105066617329915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,6144,0.031118933359781904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,65536,0.17436480522155762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,5120,0.02646613319714864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,5120,0.02823893427848816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,16384,0.04831039905548096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,4096,0.021795199314753214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,4096,0.024936532974243163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,51200,0.13736640612284343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,3584,0.02219840089480082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,3584,0.024026666084925333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,12288,0.03785813252131144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,3072,0.01718399922053019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,3072,0.02274880011876424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,6144,0.02157333294550578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,2560,0.014702933033307395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,2560,0.021577600638071695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,10240,0.0325653334458669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,2048,0.011967999736467998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,2048,0.020422399044036865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,7168,0.024129066864649454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,1536,0.009744000434875489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,8192,0.026707200209299724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,1536,0.018869332472483315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,1024,0.0076341331005096436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,1024,0.016125866770744325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,4096,0.015759999553362526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,5120,0.018822399775187175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,768,0.006651733318964641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,768,0.015687466661135355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,512,0.004354133208592733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,3072,0.01295360028743744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,3584,0.014379733800888061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,256,0.003737599899371465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,256,0.015314132968584696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,1536,0.009130666653315227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,128,0.0034229333202044168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,128,0.014892799655596414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,2560,0.01197760005791982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,64,0.0031648000081380212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,64,0.015333333611488342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,32,0.003532800078392029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,32,0.015442132949829102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,2048,0.010503466924031575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,65536,0.26573333740234373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,65536,0.1692373275756836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,51200,0.19915733337402344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,51200,0.1375989278157552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,512,0.015677866339683533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,16384,0.06937493483225504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,16384,0.05301546653111776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,1024,0.006983466446399689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,12288,0.0555189331372579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,12288,0.04331200122833252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,512,0.0057087997595469155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,10240,0.047866666316986085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,10240,0.037750399112701415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,768,0.006298666695753734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,256,0.005369600156943003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,128,0.005256533126036326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,8192,0.0339903990427653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,8192,0.03349013328552246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,7168,0.03025066653887431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,7168,0.031228800614674885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,6144,0.02686186631520589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,6144,0.02839786609013875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,65536,0.1583306630452474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,5120,0.022504534324010214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,5120,0.026042666037877398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,12288,0.034422401587168375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,4096,0.018692266941070557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,4096,0.023400533199310302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,16384,0.044062932332356766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,3584,0.016615466276804606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,3584,0.022578134139378866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,51200,0.1260981321334839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,3072,0.015152000387509666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,3072,0.021497599283854165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,10240,0.029654399553934736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,2560,0.012300800283749897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,2560,0.02058773239453634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,6144,0.01989439924558004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,7168,0.02213653326034546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,2048,0.010412800312042236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,2048,0.019542400042215982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,8192,0.024732800324757893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,1536,0.008692266543706258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,1536,0.017399466037750243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,1024,0.0068693334857622785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,1024,0.0162090669075648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,3072,0.012177067001660664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,768,0.005555200080076853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,768,0.016124799847602844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,5120,0.01741546591122945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,512,0.004043733328580856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,512,0.015635200341542563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,4096,0.014674133062362671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,256,0.0036906667053699495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,256,0.015221333503723145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,3584,0.013382400075594584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,128,0.00340693344672521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,128,0.015035733580589294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,1536,0.008198399841785432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,64,0.003193599979082743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,64,0.015003732840220132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,32,0.0035786665976047516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,2560,0.01130560040473938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,32,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,2048,0.009914666414260864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,65536,0.20489066441853843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,51200,0.16253973642985026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,51200,0.1167903979619344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,65536,0.14352960586547853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,16384,0.05957866509755453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,16384,0.04643306732177734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,128,0.005285333096981049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,512,0.005751466751098633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,768,0.006305066744486491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,12288,0.04001173178354899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,256,0.005543466905752818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,12288,0.03834240039189656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,10240,0.03401493231455485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,8192,0.028074665864308672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,8192,0.03052373329798381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,1024,0.006737066805362702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,7168,0.024999467531840007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,7168,0.027896533409754436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,10240,0.03456746737162272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,6144,0.022642133633295695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,6144,0.02547840078671773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,65536,0.13718934059143068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,5120,0.019399466117223103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,5120,0.024245333671569825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,51200,0.10934826532999675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,4096,0.017628800868988038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,4096,0.022321067253748574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,16384,0.03712533315022786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,3584,0.01381653348604838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,3584,0.021350399653116862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,10240,0.024612265825271606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,3072,0.012237866719563801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,3072,0.02065066695213318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,8192,0.02081813414891561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,2560,0.01062506635983785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,2560,0.019896533091862997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,7168,0.01857066750526428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,2048,0.009269332885742188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,2048,0.01870186726252238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,12288,0.028905600309371948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,1536,0.00784853349129359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,1536,0.016810667514801026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,6144,0.016797866423924765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,1024,0.006226133306821187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,1024,0.015849600235621132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,5120,0.014753066500027976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,768,0.004911999901135763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,768,0.01593066652615865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,4096,0.01300159990787506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,512,0.004206933577855428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,512,0.015692800283432007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,3072,0.010890666643778484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,256,0.0036629334092140196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,256,0.015210666259129844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,3584,0.012106666962305706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,2048,0.009173333644866943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,128,0.0033290666838486993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,128,0.01502826710542043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,64,0.00314026673634847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,64,0.015284267067909241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,32,0.003421866645415624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,32,0.015390933553377787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,1024,0.0063360000650088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,65536,0.17885759671529133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,2560,0.01037013332049052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,65536,0.13063573042551677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,51200,0.14193600018819172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,51200,0.1070207993189494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,16384,0.05243519941965739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,16384,0.04291946490605672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,1536,0.007513600091139476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,12288,0.03620479901631673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,768,0.006031999985376993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,12288,0.03594026565551758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,512,0.005815466741720835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,10240,0.030539733171463013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,10240,0.0329365332921346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,256,0.005300266544024149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,8192,0.024918399254480996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,8192,0.029309866825739543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,128,0.005058133105436961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,7168,0.022638932863871256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,7168,0.026342399915059406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,6144,0.020252799987792967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,6144,0.024730666478474935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,65536,0.12670079867045086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,5120,0.019155200322469076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,16384,0.03466879924138387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,5120,0.02325973312060038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,4096,0.013883733749389648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,4096,0.02185279925664266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,51200,0.09988160133361816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,3584,0.012723199526468911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,3584,0.02126400073369344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,12288,0.02750399907430013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,3072,0.011771733562151592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,3072,0.02024959921836853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,10240,0.023242666323979696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,2560,0.010114133358001709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,2560,0.019542400042215982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,8192,0.019749333461125694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,2048,0.008807466427485148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,2048,0.01826559901237488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,7168,0.017488000790278117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,1536,0.007624533275763195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,1536,0.016340266664822897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,6144,0.015852800011634825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,1024,0.005689600110054016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,1024,0.01646080017089844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,5120,0.013965866963068643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,768,0.004837333162625631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,768,0.015936000148455302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,4096,0.012597333391507468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,512,0.004154666761557261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,512,0.016201600432395935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,3072,0.010744532942771912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,256,0.003553066651026408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,256,0.015213867028554281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,3584,0.011559466520945232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,128,0.0033802665770053864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,128,0.015058133006095886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,2048,0.008293333152929943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,64,0.003180799881617228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,1024,0.006281599899133046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,64,0.014842666188875833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,32,0.0032970666885375976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,32,0.014830933014551798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,65536,0.1169269323348999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,2560,0.010013866424560546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,1536,0.007359999914964039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,51200,0.12100693384806316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,51200,0.09607040087381999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,16384,0.043884801864624026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,16384,0.04044906695683797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,65536,0.15271679560343426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,768,0.0058218667904535925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,12288,0.03440746863683065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,512,0.005717333157857259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,10240,0.030766934156417847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,10240,0.030083199342091877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,12288,0.03868693510691325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,8192,0.022502400477727256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,8192,0.026283733050028485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,256,0.0053951998551686605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,7168,0.020530132452646892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,128,0.005106133222579956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,7168,0.025111466646194458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,6144,0.018091734250386557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,6144,0.024029866854349772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,65536,0.13128639856974283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,5120,0.016620799899101257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,5120,0.022322134176890055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,16384,0.03598719835281372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,4096,0.012384000420570373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,12288,0.02773226698239644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,4096,0.021233065923055013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,3584,0.011317333579063416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,3584,0.02058026591936747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,51200,0.10435413519541423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,3072,0.01087679962317149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,3072,0.02007466753323873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,10240,0.023756800095240276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,2560,0.009457066655158997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,2560,0.018847999970118205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,6144,0.016313599546750386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,2048,0.00843519965807597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,2048,0.01763520042101542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,8192,0.01985599994659424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,1536,0.007179733117421467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,7168,0.017885865767796834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,1536,0.016531200210253397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,1024,0.005293866495291392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,1024,0.016267733772595723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,5120,0.014461867014567056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,768,0.0046293333172798155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,768,0.01567359964052836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,3072,0.010678399602572124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,512,0.004129066566626231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,512,0.015398400028546652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,3584,0.011597866813341778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,256,0.0036159999668598174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,256,0.015005866686503092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,2560,0.00965013305346171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,128,0.003205333401759466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,128,0.014971733093261719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,4096,0.012713600198427835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,64,0.0030464000999927522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,64,0.01488746702671051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,32,0.003291733314593633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,1536,0.007294933497905731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,32,0.014732799927393594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,65536,0.1311082681020101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,65536,0.10577387015024822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,51200,0.10386453469594319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,2048,0.008414933085441589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,51200,0.08714346885681153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,16384,0.03928320010503133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,16384,0.03936853408813477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,768,0.006007466713587443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,12288,0.031007999181747438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,256,0.005277866621812185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,12288,0.031906133890151976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,1024,0.006401066482067108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,10240,0.027448532978693645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,10240,0.028679466247558592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,512,0.0056874667604764305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,8192,0.02249280015627543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,8192,0.025139200687408447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,7168,0.018367999792099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,7168,0.02500266631444295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,128,0.005125333368778229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,6144,0.01604693333307902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,6144,0.023386667172114052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,65536,0.12645119825998943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,5120,0.013651200135548911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,5120,0.02182933290799459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,16384,0.034358398119608564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,4096,0.011336533228556316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,4096,0.02030613422393799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,51200,0.10219199657440185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,3584,0.010482133428255717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,3584,0.01986133257548014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,12288,0.02635093331336975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,3072,0.009543466567993163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,3072,0.018515199422836304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,10240,0.023101866245269775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,2560,0.008572799960772197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,2560,0.019250132640202842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,6144,0.015574399630228677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,2048,0.007569066683451335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,2048,0.017146666844685875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,8192,0.01904319922129313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,1536,0.006403199831644694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,1536,0.017117865880330405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,7168,0.017425066232681273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,1024,0.004822400212287903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,1024,0.01584106683731079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,4096,0.012402133146921793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,768,0.004317866762479147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,768,0.015542399883270264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,5120,0.014083199699719749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,512,0.00382080003619194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,512,0.015410133202870688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,3584,0.011347200473149617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,256,0.003390933324893316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,256,0.014990933736165366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,3072,0.00974506636460622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,128,0.0031701333820819853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,128,0.014929067095120749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,1536,0.0070816000302632645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,64,0.0029343999922275543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,64,0.014740266402562461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,2560,0.009307733178138733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,32,0.003028266628583272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,32,0.014830933014551798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,65536,0.10783572991689046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,65536,0.09398187001546224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,51200,0.0856607993443807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,51200,0.07693119843800864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,2048,0.00792746643225352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,1024,0.006467199822266896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,16384,0.03271999955177307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,16384,0.03488959868748982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,12288,0.02534826596577962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,12288,0.03081386685371399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,256,0.005161599814891815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,512,0.005628799895445505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,10240,0.022550400098164877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,768,0.00603413333495458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,8192,0.01831573247909546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,10240,0.028862933317820232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,8192,0.025432533025741576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,128,0.005055999755859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,7168,0.01644480029741923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,7168,0.023810132344563802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,65536,0.10392213662465413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,6144,0.015660799543062844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,6144,0.023118933041890465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,5120,0.013361066579818726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,51200,0.082259202003479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,16384,0.02850346763928731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,4096,0.010707199573516846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,4096,0.01995519995689392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,12288,0.022247467438379923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,3584,0.009890133142471313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,3584,0.019639466206232706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,5120,0.021038933595021566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,3072,0.009117866555849712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,3072,0.018116267522176106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,8192,0.016321067015329996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,2560,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,2560,0.017655466000239053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,7168,0.014994133512179056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,2048,0.006977066894372304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,2048,0.017199999094009398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,10240,0.01981333295504252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,1536,0.005493333439032236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,1536,0.015867732961972556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,6144,0.014030933380126953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,1024,0.004621866842110952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,1024,0.015682133038838704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,5120,0.012456533312797547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,768,0.0041184000670909885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,768,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,4096,0.011059199770291645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,512,0.003731200098991394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,512,0.015474133690198264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,3584,0.010215466221173603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,256,0.0032405334214369455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,256,0.014900267124176025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,3072,0.008754133184750875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,128,0.0030559999247392017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,128,0.014678399761517844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,2560,0.008354133367538452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,64,0.002917333443959554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,64,0.01471573313077291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,32,0.0029824001093705496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,2048,0.0073845331867535904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,32,0.014813866217931113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,65536,0.0831285317738851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,65536,0.08049493630727132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,51200,0.06687146822611491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,51200,0.06648213466008504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,1536,0.00705813318490982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,16384,0.02521173357963562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,16384,0.030501333872477214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,1024,0.006118399898211161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,12288,0.020521599054336547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,12288,0.027617067098617554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,768,0.0056981335083643595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,10240,0.01786880095799764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,10240,0.025069866577784223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,512,0.005350400010744731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,8192,0.01447466711203257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,8192,0.023337600628534953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,128,0.0050122668345769245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,7168,0.013315199812253316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,256,0.005061333378156027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,7168,0.02243946592013041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,6144,0.012552533547083536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,6144,0.02185066739718119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,65536,0.1023957331975301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,5120,0.011073066790898641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,5120,0.02107306718826294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,51200,0.08042026360829671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,4096,0.009689600268999735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,4096,0.01877546707789103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,16384,0.027530666192372637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,3584,0.009008000294367473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,3584,0.018367999792099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,12288,0.021691733598709108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,3072,0.008297599852085114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,3072,0.01835093299547831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,10240,0.018897066513697304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,2560,0.007378133138020833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,2560,0.017590399583180746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,8192,0.01618133286635081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,2048,0.006451199948787689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,2048,0.016963199774424235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,6144,0.013685333728790283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,1536,0.005499733487764994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,7168,0.01474666694800059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,1536,0.015921066204706825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,1024,0.004579199850559235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,1024,0.015547733505566916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,5120,0.012206932902336121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,768,0.004167466859022776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,768,0.015432533621788026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,4096,0.010300800204277039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,512,0.003701333453257879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,512,0.014978133638699851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,3072,0.00867306689421336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,3584,0.009685333569844563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,256,0.003272533416748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,256,0.014638933539390563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,2048,0.007256533205509186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,128,0.002995199958483378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,128,0.014840533336003622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,2560,0.008208000163237254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,64,0.002935466667016347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,64,0.014632532993952433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,32,0.0030037333567937215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,32,0.014683733383814493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,1024,0.006061866879463196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,65536,0.058517332871754965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,65536,0.06888213157653808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,65536,0.10179626941680908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,51200,0.04750613371531169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,51200,0.056491732597351074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,16384,0.01866453289985657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,16384,0.02637653350830078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,16384,0.027046400308609008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,12288,0.014355199535687766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,12288,0.023431466023127238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,1536,0.006913066903750102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,10240,0.01263146698474884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,12288,0.021248000860214233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,10240,0.023731199900309245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,8192,0.01084266702334086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,8192,0.021849600474039714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,8192,0.01602133313814799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,7168,0.012013866504033407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,7168,0.021172267198562623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,6144,0.009719467163085938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,6144,0.020631466309229532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,6144,0.012807466586430869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,5120,0.009576533238093059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,5120,0.019638399283091225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,768,0.005657599866390228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,512,0.005402666827042898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,4096,0.008545066912968953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,4096,0.010044800241788228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,4096,0.018615466356277467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,3584,0.008392533659934998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,3584,0.018253866831461588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,3584,0.009417600433031718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,3072,0.007859200239181519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,3072,0.01769386728604635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,3072,0.008458667000134786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,2560,0.006635733445485433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,2560,0.01716586748758952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,2048,0.005618133147557576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,2048,0.01665173371632894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,2048,0.007154133419195812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,1536,0.005002666513125102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,51200,0.07930346330006918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,1536,0.016057599584261575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,1024,0.0042805333932240805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,1024,0.015668267011642457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,256,0.004980266590913137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,768,0.003866666555404663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,10240,0.01875093380610148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,768,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,512,0.0035146666069825493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,768,0.005746133128801982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,512,0.01516800026098887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,256,0.0032085334261258446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,256,0.014546133081118264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,7168,0.014131200313568116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,2560,0.008039466540018718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,128,0.0030154667794704436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,128,0.014668800433476768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,64,0.002794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,64,0.014607999722162882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,32,0.0029205332199732465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,32,0.014625066518783569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,65536,0.04646613200505574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,65536,0.06194026470184326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,65536,0.098798934618632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,51200,0.03834026654561361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,128,0.004816000163555145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,51200,0.05200426578521729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,16384,0.014637866616249084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,51200,0.07619520028432211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,16384,0.0253493328889211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,12288,0.011848533153533935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,16384,0.026739199956258137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,12288,0.021132800976435342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,12288,0.022719999154408775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,10240,0.01151039997736613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,5120,0.011382399996121725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,10240,0.021592533588409422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,10240,0.018196266889572144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,8192,0.01011306643486023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,8192,0.020459733406702676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,7168,0.009725866715113322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,8192,0.015191466609636942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,6144,0.008941866954167684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,7168,0.013722667098045349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,6144,0.012488533059755962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,6144,0.019809067249298096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,5120,0.008899199962615966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,5120,0.019053866465886436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,1536,0.006866133213043213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,4096,0.007573333382606506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,4096,0.01856213410695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,1024,0.00595306654771169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,4096,0.010002133250236512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,3584,0.008210133512814839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,3584,0.018155733744303383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,3584,0.009264000256856282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,3072,0.0069578667481740315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,3072,0.017486933867136636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,3072,0.008455466230710347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,2560,0.006202666461467743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,2560,0.008067200084527333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,2560,0.017375999689102174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,2048,0.005558399856090546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,512,0.005193600058555603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,2048,0.01662506659825643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,2048,0.00710399995247523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,1536,0.004841599861780802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,1536,0.006797866523265838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,1536,0.01602453291416168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,7168,0.020680532852808634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,768,0.0038431999584039056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,1024,0.015617066621780395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,256,0.0049674664934476215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,768,0.005421866476535797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,768,0.01551253298918406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,512,0.003470933437347412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,256,0.003101866692304611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,512,0.015134933590888976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,256,0.004829866687456766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,256,0.014666666587193808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,128,0.0028575999041398365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,128,0.004825599988301595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,128,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,64,0.002754133443037669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,32,0.002737066646416982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,64,0.014442666371663412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,32,0.014626133441925048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,65536,0.03455573320388794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,65536,0.05578133265177408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,51200,0.028535467386245728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,51200,0.0460480014483134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,51200,0.07276906967163085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,16384,0.012145066261291504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,16384,0.023474133014678954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,1024,0.004186666508515676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,12288,0.010123733679453533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,12288,0.022913066546122234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,12288,0.02062506675720215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,10240,0.009841066598892213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,10240,0.02119893431663513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,10240,0.01802240014076233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,128,0.004739200075467428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,8192,0.020387200514475505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,7168,0.008102400104204814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,5120,0.011308800180753071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,7168,0.02016426722208659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,7168,0.013698133826255798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,6144,0.009154133001963298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,6144,0.01955946683883667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,5120,0.008100266754627227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,5120,0.01972800095876058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,5120,0.011159466703732808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,4096,0.007097599903742473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,4096,0.018755199511845906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,4096,0.010005333026250203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,3584,0.006741333504517872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,3584,0.018374399344126383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,3584,0.009322667121887207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,3072,0.006411733229955037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,3072,0.017821866273880004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,3072,0.00842133363087972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,2560,0.006296533346176148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,8192,0.008381866415341695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,2560,0.01727573275566101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,2560,0.008057599763075511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,1024,0.005850666761398315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,2048,0.0055871998270352686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,512,0.00522986650466919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,1536,0.004882133503754934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,2048,0.016834133863449098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,1536,0.01601599951585134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,1024,0.004203733305136363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,1024,0.005809066692988077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,768,0.0037941334148248037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,768,0.015943466623624166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,65536,0.09187093575795492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,512,0.0034656000634034475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,512,0.014998400211334228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,512,0.005176533261934916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,256,0.0031530665854612983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,256,0.014599466323852539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,16384,0.026446932554244997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,128,0.0028949332733949023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,128,0.01474240024884542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,128,0.004675200084845225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,64,0.002791466563940048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,64,0.014929067095120749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,1024,0.015828266739845276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,32,0.0030752000709374744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,32,0.014957867066065469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,65536,0.023089067141215006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,8192,0.014816000064214071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,65536,0.04743573268254598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,6144,0.012541866302490235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,51200,0.01900906761487325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,51200,0.037656533718109134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,51200,0.07121067047119141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,16384,0.00842026670773824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,16384,0.025154133637746174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,16384,0.021205333868662517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,12288,0.008429867029190064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,12288,0.02031573255856832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,12288,0.019942400852839152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,10240,0.007482666770617168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,10240,0.020086399714152017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,8192,0.006723199784755707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,8192,0.019169066349665323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,8192,0.014682666460673014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,7168,0.006428800026575724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,7168,0.02000746726989746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,7168,0.013459199666976928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,6144,0.006232533355553945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,6144,0.018794665733973183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,2048,0.0071712002158164975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,5120,0.006631466746330261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,5120,0.019170133272806804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,5120,0.011100799838701884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,4096,0.006212266782919565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,4096,0.01839039921760559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,4096,0.009995733698209126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,768,0.00543146679798762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,3584,0.006609066824118296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,3584,0.017909334103266398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,3072,0.006297599772612255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,10240,0.0173632005850474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,3072,0.017511467138926186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,3072,0.008424533406893413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,2560,0.006065066655476888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,2560,0.01684160033861796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,2048,0.0054624001185099285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,2048,0.016383999586105348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,2048,0.007187200089295705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,256,0.004880000154177348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,1536,0.004799999793370565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,1536,0.006694399813810985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,1536,0.01562346617380778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,1024,0.0040501333773136135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,1024,0.015528532862663268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,768,0.003673599908749262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,768,0.015321600437164306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,768,0.005374933282534281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,512,0.0034175999462604523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,512,0.014987732966740927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,1536,0.006725333134333293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,512,0.005153066913286845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,256,0.0030303999781608583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,256,0.004833066463470459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,128,0.002869333326816559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,256,0.014838400483131408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,64,0.0026911998788515727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,128,0.004745600124200186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,128,0.014501333236694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,32,0.002733866622050603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,64,0.01462399959564209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,32,0.014459733168284097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,65536,0.013208533326784769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,65536,0.04135359923044841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,51200,0.011619200309117634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,3584,0.009366400043169658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,65536,0.09215146700541178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,16384,0.006938666601975759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,51200,0.07252693176269531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,16384,0.021492266654968263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,16384,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,12288,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,10240,0.006470400094985962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,12288,0.020151466131210327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,10240,0.017565866311391197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,10240,0.019723733266194664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,8192,0.006427733103434245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,8192,0.014673067132631936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,8192,0.01927679975827535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,7168,0.006298666695753734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,7168,0.020600533485412596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,7168,0.01357439955075582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,6144,0.006106666723887126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,6144,0.019180800517400107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,5120,0.00645653357108434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,5120,0.01936960021654765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,51200,0.03351786533991496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,4096,0.00609493354956309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,4096,0.018433066209157307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,12288,0.019935999313990274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,4096,0.009783466657002766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,3584,0.017972266674041747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,3584,0.009250133236249288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,3072,0.006190933287143707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,65536,0.09097493489583333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,3072,0.017427200078964235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,3072,0.008380800485610962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,2560,0.006045866509278615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,2560,0.01708266735076904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,2560,0.007973333199818928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,3584,0.006525866687297821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,2048,0.016673066218694053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,2048,0.007124266525109608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,1536,0.004775466521581014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,1536,0.01585599978764852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,6144,0.01234346628189087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,1536,0.00680320014556249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,1024,0.004007466634114583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,1024,0.015613866845766702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,1024,0.005940266450246175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,768,0.003703466554482778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,768,0.005401599903901418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,512,0.00340693344672521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,512,0.014995200435320535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,2048,0.005404800176620483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,256,0.003031466652949651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,256,0.014833066860834757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,1024,0.005858133236567179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,256,0.004841599861780802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,128,0.014716800053914389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,128,0.004643199841181437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,64,0.002682666728893916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,64,0.014620799819628397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,768,0.015014400084813436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,32,0.0027082666754722597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,65536,0.010641066233317058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,32,0.014452266693115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,51200,0.00914026697476705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,65536,0.038284798463185624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,2560,0.007982933521270752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,16384,0.006311466793219249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,12288,0.006246399879455566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,16384,0.02137813369433085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,10240,0.006450133522351582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,12288,0.019824000199635823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,8192,0.006387199958165486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,10240,0.020129066705703736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,8192,0.019230933984120686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,6144,0.006017066538333893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,7168,0.019342933098475137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,5120,0.006397866706053417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,6144,0.019339734315872194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,4096,0.005977599819501241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,5120,0.0194815993309021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,3584,0.006514133512973785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,4096,0.018178133169809978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,3584,0.01809813380241394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,3072,0.006131199995676676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,2560,0.0060576001803080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,3072,0.017504000663757326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,2560,0.016922666629155477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,2048,0.016705065965652466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,1536,0.0046965335806210835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,128,0.0028661333024501802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,1536,0.016023466984430947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,1024,0.015503999590873719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,768,0.003705599904060364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,768,0.015400532881418863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,512,0.0033919999996821085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,6144,0.012466133634249369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,512,0.015007999539375306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,5120,0.011265066266059876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,128,0.0028746667007605235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,256,0.01467519998550415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,64,0.0026837334036827086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,128,0.014476799964904785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,51200,0.031228800614674885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,64,0.014640000462532044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,32,0.002739199995994568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,65536,0.009236266215642292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,32,0.014535466829935709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,51200,0.00874133308728536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,65536,0.036968533198038736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,16384,0.006196266909440359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,51200,0.030418133735656737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,12288,0.006078933179378509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,7168,0.006185600161552429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,16384,0.021734400590260824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,12288,0.02003413240114848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,8192,0.006232533355553945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,10240,0.01965226729710897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,8192,0.01888426740964254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,6144,0.005948799848556519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,7168,0.01901866594950358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,5120,0.0063360000650088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,6144,0.018781866629918417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,4096,0.005757866799831391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,5120,0.019740800062815346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,3584,0.006229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,4096,0.01823893388112386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,2048,0.005393066505591074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,3072,0.005899733304977417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,3584,0.017641599973042807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,2560,0.006045866509278615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,3072,0.017157334089279174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,2560,0.01747093399365743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,2048,0.005389866729577383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,1536,0.004738133152325948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,1024,0.003977599988381068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,1024,0.004001066585381826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,1536,0.016114133596420287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,768,0.0037280000746250153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,1024,0.016300800442695617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,512,0.0033183999359607695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,768,0.01513920029004415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,256,0.0030752000709374744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,512,0.015089066823323569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,128,0.002825599908828735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,256,0.014627200365066529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,64,0.002630399912595749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,256,0.003005866706371307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,128,0.014380799730618796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,32,0.002614400039116542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,64,0.014604799946149192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,32,0.014317867159843446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,10240,0.006284800171852112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,7168,0.0060479998588562015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,512,0.005162666738033295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,2048,0.0164000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,16384,0.40830081303914384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,16384,0.7653013229370117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,16384,0.3748010635375977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,12288,0.5831530888875325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,12288,0.3127456029256185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,51200,1.1321792602539062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,10240,0.48693014780680344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,51200,1.2220928192138671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,10240,0.26421759923299154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,10240,0.23826773961385092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,8192,0.21615893046061196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,8192,0.22218987147013344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,7168,0.19036800066630047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,7168,0.34105920791625977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,8192,0.4171317418416341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,6144,0.2964639981587728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,51200,2.3862154642740885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,6144,0.15011946360270184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,6144,0.1675615946451823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,5120,0.24808319409688315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,5120,0.16168959935506183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,4096,0.11924479802449543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,4096,0.10505599975585937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,3584,0.1076639970143636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,3584,0.17726507186889648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,4096,0.23156159718831382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,3584,0.09466453393300375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,3072,0.09560213088989258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,3072,0.15324053764343262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,3072,0.08256746927897135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,2560,0.08306986490885417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,2560,0.1469770590464274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,2560,0.07174506982167562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,2048,0.10590613683064778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,2048,0.07070506413777669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,1536,0.05821653207143148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,1536,0.09210453033447266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,1536,0.05233386754989624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,1024,0.05758826732635498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,768,0.045348266760508224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,1024,0.03852800130844116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,1024,0.04925653139750163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,768,0.03905386527379354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,512,0.036798934141794845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,512,0.03160319924354553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,512,0.02442880074183146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,256,0.020940800507863365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,256,0.018515199422836304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,256,0.024175999561945598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,128,0.011306666334470113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,128,0.016383999586105348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,64,0.008713600039482117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,128,0.02146880030632019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,65536,32,0.008004266520341237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,64,0.020190932353337607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,65536,32,0.020480000972747804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,12288,0.3095285415649414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,2048,0.059779198964436855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,7168,0.17216426531473797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,5120,0.13060692946116131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,65536,1.2270591735839844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,51200,0.9693781534830729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,16384,0.6063466389973958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,16384,0.374895985921224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,65536,768,0.03171626726786296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,65536,2.3670015970865887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,51200,1.8938677469889325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,12288,0.46164585749308273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,12288,0.25346879959106444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,10240,0.4432266553243001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,10240,0.24371093114217124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,10240,0.2058645407358805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,8192,0.311518923441569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,8192,0.19885973930358886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,7168,0.27204694747924807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,7168,0.1596949259440104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,6144,0.2385045369466146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,6144,0.13839680353800457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,65536,1.1597994486490886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,5120,0.22842559814453126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,5120,0.11844800313313801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,51200,0.906222915649414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,16384,0.3185237248738607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,12288,0.22940373420715332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,4096,0.10255359808603923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,4096,0.18387626012166342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,3584,0.14052906036376953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,3584,0.0920469363530477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,3072,0.1235968033472697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,3072,0.07976960341135661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,8192,0.15667200088500977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,2560,0.10458239714304607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,2560,0.06948479811350504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,7168,0.13905067443847657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,2048,0.08621226946512858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,2048,0.05971200068791708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,1536,0.06593600114186605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,1536,0.05038293202718099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,1536,0.03884586493174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,1024,0.04746880133946736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,6144,0.12076906363169353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,1024,0.04020586808522542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,768,0.037774932384490964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,768,0.034626134236653644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,5120,0.10246187051137287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,512,0.030027733246485395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,512,0.028550400336583452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,4096,0.08462719917297364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,256,0.017921066284179686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,3584,0.07484479745229086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,256,0.023205333948135377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,128,0.010703999797503154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,128,0.020901334285736085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,3072,0.06607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,64,0.008269866804281871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,64,0.020068265994389854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,51200,32,0.00783253312110901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,2560,0.05740799903869629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,51200,32,0.020141865809758505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,2048,0.04706986745198567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,1024,0.032290132840474446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,65536,0.4094549179077148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,768,0.02494293252627055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,65536,0.3784576098124186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,65536,0.7797034581502278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,16384,0.19105812708536785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,16384,0.13266239960988363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,51200,0.32590293884277344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,51200,0.29656960169474283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,16384,0.10322133700052898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,51200,0.5830591837565104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,12288,0.14587093989054362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,10240,0.12263466517130535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,12288,0.09402026335398356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,10240,0.0668351968129476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,8192,0.09870933691660563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,10240,0.08952853679656983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,8192,0.06699732939402261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,8192,0.05534293254216512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,7168,0.08666986624399821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,7168,0.061196800072987875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,6144,0.07578453222910563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,6144,0.05529599984486898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,6144,0.04407680034637451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,5120,0.0639135996500651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,5120,0.04897706508636475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,512,0.01935466726620992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,5120,0.03823146820068359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,4096,0.05595200061798096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,4096,0.04274026552836101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,3584,0.046187734603881835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,3584,0.03947093486785889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,3584,0.029526400566101074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,3072,0.04069013198216756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,3072,0.026537599166234334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,3072,0.03633066813151042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,2560,0.03488746484120687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,256,0.015007999539375306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,2048,0.028809599081675213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,2560,0.023414399226506552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,2560,0.03284800052642822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,2048,0.029128533601760865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,2048,0.019617066780726115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,1536,0.023137066761652628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,1536,0.025370667378107708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,1536,0.016119466225306193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,1024,0.016075733304023742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,1024,0.02198293407758077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,768,0.012570666273434958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,1024,0.012943999965985617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,512,0.009476266304651896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,768,0.01165226697921753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,512,0.018910932540893554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,512,0.009943466385205586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,256,0.007392000158627827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,256,0.016516266266504924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,128,0.004554666578769684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,128,0.016155733664830526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,128,0.007204266885916392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,64,0.004195199906826019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,64,0.0164682666460673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,32,0.00489279975493749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,32,0.016406400005022685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,4096,0.03222506642341614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,16384,768,0.02063573400179545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,65536,0.580673090616862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,16384,256,0.006666666766007741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,65536,0.31230398813883464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,51200,0.24885546366373695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,51200,0.23118720054626465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,12288,0.07911360263824463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,51200,0.4587413469950358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,16384,0.14186986287434894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,16384,0.09143573443094889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,16384,7168,0.05004693269729614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,12288,0.12657492955525715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,12288,0.07334293524424235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,12288,0.06178026596705118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,10240,0.09184319972991943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,10240,0.06380906502405802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,10240,0.053107198079427084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,8192,0.07385493119557698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,8192,0.04382293224334717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,7168,0.06509439945220948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,7168,0.04985599915186564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,7168,0.03946559826533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,6144,0.057214931646982825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,6144,0.04557120005289714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,5120,0.04860266844431559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,51200,128,0.012681600451469422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,5120,0.04063466787338257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,5120,0.030499200026194256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,4096,0.03993599812189738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,4096,0.025862399737040204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,4096,0.0358570655186971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,3584,0.035689600308736164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,3584,0.023374933004379272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,3584,0.03347413142522176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,3072,0.031335467100143434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,2560,0.02691199978192647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,3072,0.020856533447901407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,3072,0.030777599414189654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,2048,0.022331732511520385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,2560,0.027960532903671266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,2048,0.015348266561826071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,1536,0.017828265825907387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,1536,0.02273919979731242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,1536,0.013132799665133157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,1024,0.012319999933242797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,1024,0.011026133100191753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,1024,0.02029013236363729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,768,0.010250666737556457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,768,0.018942934274673463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,768,0.009804800152778625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,512,0.007946666578451793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,512,0.01657919983069102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,512,0.007516799867153168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,256,0.004818133513132731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,256,0.016042666633923848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,128,0.004120533416668574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,8192,0.05445653200149536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,128,0.006030933558940887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,128,0.016557866334915163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,64,0.016729599237442015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,32,0.004304000238577525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,32,0.01684373418490092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,12288,2048,0.02490880091985067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,65536,0.49907414118448895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,65536,0.2932917277018229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,65536,0.29241065979003905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,16384,0.08045226732889811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,65536,0.2542805353800456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,16384,0.13031786282857258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,51200,0.23513174057006836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,6144,0.035010135173797606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,51200,0.3951818784077962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,16384,0.08050879637400309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,12288,0.09255999724070231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,12288,0.07266773382822672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,10240,0.07754240036010743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,10240,0.05720106760660807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,10240,0.046130132675170896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,8192,0.06290453275044759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,8192,0.04931519826253255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,7168,0.055718398094177245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,2560,0.018742400407791137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,7168,0.045393065611521406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,7168,0.03440426588058472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,6144,0.04901119867960612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,6144,0.04123520056406657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,5120,0.04158399899800618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,12288,64,0.003866666555404663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,5120,0.036830933888753255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,4096,0.03410346508026123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,4096,0.032790400584538776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,12288,256,0.006419200201829274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,4096,0.02242986758550008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,3584,0.030435200532277423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,3584,0.03033706744511922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,3584,0.020115200678507486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,3072,0.026866134007771807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,3072,0.027770666281382243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,3072,0.01787839929262797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,2560,0.023130667209625245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,8192,0.03840426603953044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,2560,0.025422932704289754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,2048,0.01914773384730021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,2048,0.023460266987482707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,2048,0.013687466581662497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,1536,0.014983466267585755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,1536,0.021692800521850585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,1536,0.011682132879892986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,1024,0.010787199934323628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,1024,0.0192138671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,51200,0.19901760419209796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,768,0.010062932968139648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,16384,0.06933973630269369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,768,0.008745599786440532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,768,0.018823466698328652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,12288,0.05352533260981242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,512,0.007044266661008198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,256,0.0046165332198143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,512,0.006725333134333293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,512,0.0165503998597463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,128,0.004067199925581614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,256,0.0159061332543691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,128,0.015569067001342774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,128,0.0056096002459526065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,64,0.0037248000502586366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,64,0.015619200468063355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,10240,32,0.003864533454179764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,10240,32,0.015607466300328573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,6144,0.030294400453567506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,5120,0.02634773254394531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,65536,0.3859583854675293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,51200,0.3064319928487142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,51200,0.1892159938812256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,16384,0.10698986848195394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,16384,0.06780906518300375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,12288,0.07365653514862061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,12288,0.05558293263117472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,2560,0.01581013302008311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,10240,0.06217813491821289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,10240,0.049356798330942794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,65536,0.2340384006500244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,8192,0.054843731721242274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,8192,0.0422111988067627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,1024,0.010141866405804952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,7168,0.044640000661214194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,7168,0.03900586764017741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,10240,256,0.005980800092220307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,6144,0.03973546822865804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,6144,0.035860268274943034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,65536,0.2141536076863607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,5120,0.033411200841267905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,5120,0.03258986671765645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,16384,0.05797013441721598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,4096,0.027831466992696126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,4096,0.028730666637420653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,12288,0.045159467061360675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,3584,0.024939733743667602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,3584,0.026961066325505573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,10240,0.038729600111643475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,3072,0.02196906606356303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,3072,0.025073067347208662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,8192,0.032707200447718306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,2560,0.018861865997314452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,2560,0.02332373261451721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,7168,0.02895786762237549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,2048,0.01564906636873881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,51200,0.17064959208170574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,2048,0.021721599499384563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,1536,0.012155733505884806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,1536,0.020411733786265054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,6144,0.0260917325814565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,1024,0.009256533781687419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,1024,0.018343466520309448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,5120,0.022771199544270836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,768,0.0077909335494041445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,768,0.01675093372662862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,4096,0.019384533166885376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,512,0.006291200220584869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,512,0.016455466548601784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,3584,0.01798080007235209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,256,0.004337066908677419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,256,0.01564586659272512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,3072,0.01574399967988332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,128,0.0036906667053699495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,128,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,2560,0.014263466993967692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,64,0.0035285333792368567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,64,0.015629866719245912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,8192,32,0.003920000046491623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,8192,32,0.015635200341542563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,2048,0.012662399808565775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,1536,0.011271466811498005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,65536,0.36067625681559246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,1024,0.009124267101287841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,65536,0.2093045393625895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,51200,0.16906453768412272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,51200,0.2787285486857096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,16384,0.09889492988586426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,768,0.007771733403205872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,16384,0.06191573143005371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,12288,0.07489706675211588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,12288,0.05120000044504801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,512,0.006804266571998596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,10240,0.055034665266672764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,10240,0.04607466856638591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,256,0.006246399879455566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,8192,0.044599465529123944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,8192,0.039496533075968426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,8192,128,0.005798399945100148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,7168,0.039572266737620036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,7168,0.03582079807917277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,6144,0.035230934619903564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,6144,0.03359253406524658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,65536,0.20478827158610025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,5120,0.03382399876912435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,5120,0.030074665943781536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,16384,0.05484586556752523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,4096,0.02464639941851298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,51200,0.16126400629679363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,4096,0.026703999439875288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,3584,0.021899733940760294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,3584,0.025012266635894776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,12288,0.042387199401855466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,3072,0.019308799505233766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,3072,0.023487999041875204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,10240,0.03648213148117065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,2560,0.016620799899101257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,2560,0.022396800915400188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,8192,0.03036053379376729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,2048,0.013398399949073792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,2048,0.020725333690643312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,7168,0.027471999327341717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,1536,0.010866133371988933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,1536,0.019428267081578573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,6144,0.024039467175801597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,1024,0.008301866551240284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,1024,0.01762346625328064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,5120,0.020878932873407998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,768,0.007065600156784058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,768,0.01579839984575907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,4096,0.01858133276303609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,3584,0.016123732924461363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,512,0.004824533561865489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,512,0.01583999991416931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,256,0.0038805333276589714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,256,0.015479466319084168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,3072,0.01456000010172526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,128,0.003432533393303553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,128,0.014961066842079162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,2560,0.013451733191808066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,64,0.0033503999312718712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,64,0.01525973379611969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,7168,32,0.00352960005402565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,7168,32,0.015273599823315939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,2048,0.011645866433779399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,65536,0.30820798873901367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,1536,0.010281599561373393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,65536,0.19907946586608888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,51200,0.24554452896118165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,51200,0.14879147211710614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,16384,0.08102933565775552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,16384,0.056601599852244056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,1024,0.008384000261624653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,12288,0.056600534915924074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,12288,0.04699840148289998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,768,0.007218133409818013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,10240,0.047926398118336995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,10240,0.04166080156962077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,10240,0.03352533181508382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,8192,0.03889386653900147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,8192,0.03662506739298503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,512,0.006353066861629486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,256,0.00581333339214325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,7168,0.034541865189870194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,7168,0.03391253153483073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,6144,0.030807467301686604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,6144,0.030920533339182536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,65536,0.18741973241170246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,5120,0.025972266991933186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,7168,128,0.005437866846720377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,5120,0.028038400411605834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,4096,0.021755733092625937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,4096,0.02498240073521932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,51200,0.14736426671346028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,3584,0.021875200668970744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,3584,0.023461333910624185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,16384,0.049652266502380374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,3072,0.017061332861582436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,3072,0.022677334149678548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,12288,0.03902293443679809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,3072,0.013358933726946512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,2560,0.014495999614397685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,2560,0.021333332856496176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,8192,0.027782400449117024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,2048,0.011918933192888895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,2048,0.020020266373952232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,2048,0.010999466975529988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,1536,0.009678933024406432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,1536,0.01909546653429667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,7168,0.02504533330599467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,1024,0.0076341331005096436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,6144,0.022312533855438233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,1024,0.016434133052825928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,768,0.006734933455785115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,768,0.015732266505559287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,5120,0.01893333395322164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,512,0.004466133316357931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,4096,0.01636799971262614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,512,0.01565333306789398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,256,0.0036501333117485045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,256,0.015093333522478738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,3584,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,128,0.0032960000137488045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,128,0.015400532881418863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,128,0.005132799843947092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,64,0.0036245333651701607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,64,0.015485866864522298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,6144,32,0.003671466559171677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,6144,32,0.015492266416549683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,65536,0.1650826613108317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,65536,0.24915199279785155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,1536,0.009448533256848652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,51200,0.19751253128051757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,1024,0.0073642666141192125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,51200,0.1344490687052409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,16384,0.0689632018407186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,16384,0.047017598152160646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,16384,0.0507861336072286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,12288,0.05533440113067627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,12288,0.04315946499506633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,10240,0.047319467862447104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,10240,0.03960106770197551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,2560,0.012284800410270691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,8192,0.033342933654785155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,8192,0.03282346725463867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,512,0.0061706667145093284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,7168,0.030525867144266767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,768,0.006903466582298279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,6144,256,0.005432533224423727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,6144,0.027167999744415285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,6144,0.028011733293533327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,65536,0.1748576005299886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,7168,0.03081386685371399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,5120,0.024991999069849648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,5120,0.025676800807317095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,4096,0.018626133600870766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,51200,0.13908054033915201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,4096,0.023411200443903605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,12288,0.036635732650756835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,3584,0.022273067633310953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,3584,0.014089600245157877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,3072,0.014387200276056925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,10240,0.031471999486287434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,3072,0.02119040091832479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,3072,0.012805333733558655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,2560,0.012335999806722005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,2560,0.020460800329844157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,2048,0.010408533612887065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,8192,0.025978666543960572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,2048,0.019433599710464478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,1536,0.00860479970773061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,1536,0.01795733372370402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,3584,0.016721065839131674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,1024,0.006884266436100006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,1024,0.016522666811943053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,7168,0.023587199052174886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,768,0.005726933479309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,6144,0.021154133478800456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,768,0.015960533420244852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,512,0.00412266676624616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,512,0.015838932991027833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,5120,0.01781866749127706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,256,0.003907199949026108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,256,0.015336533387502035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,4096,0.01532373329003652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,128,0.0034677334129810332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,128,0.01487573285897573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,2560,0.011780266960461933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,64,0.0032149332265059153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,64,0.015270400047302245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,5120,32,0.0033919999996821085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,2048,0.01056106686592102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,5120,32,0.015177599589029946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,1536,0.008453333377838134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,65536,0.20272746086120605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,51200,0.160918394724528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,51200,0.11355733076731364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,1024,0.0075445334116617845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,16384,0.05526080131530762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,16384,0.045315198103586835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,768,0.006781866649786632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,512,0.006239999830722809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,12288,0.043934933344523114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,65536,0.139411195119222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,12288,0.0378165324529012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,10240,0.03794773419698079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,10240,0.03364693323771159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,256,0.0055744002262751256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,8192,0.02752106587092082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,5120,128,0.005312000215053558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,8192,0.02972480058670044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,7168,0.0245685338973999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,7168,0.027177600065867107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,7168,0.020974934101104736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,6144,0.02216959993044535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,6144,0.025455999374389648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,65536,0.1589344024658203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,5120,0.018917334079742432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,5120,0.02369706630706787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,51200,0.1236458698908488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,4096,0.015319466590881348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,16384,0.04174720048904419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,4096,0.021964800357818604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,3584,0.013722667098045349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,3584,0.02113173405329386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,3072,0.012107732892036437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,3072,0.02032533288002014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,12288,0.032280532519022624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,2560,0.010587732990582783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,2560,0.019801600774129232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,10240,0.028228267033894854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,2048,0.009084799885749817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,2048,0.018685867389043175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,8192,0.02305493354797363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,1536,0.0076896001895268755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,1536,0.017172267039616905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,6144,0.01911466717720032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,1024,0.0062730665008227035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,1024,0.016403200229008992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,5120,0.015962666273117064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,768,0.004628266890843709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,768,0.015779200196266174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,4096,0.013821867108345032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,512,0.004110933343569437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,512,0.015415466825167336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,3584,0.013082666198412576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,256,0.00344106654326121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,256,0.015025066335995993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,3072,0.011669333775838215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,128,0.003222399950027466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,128,0.014622933665911355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,2560,0.011168000102043153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,64,0.0031776001056035364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,64,0.015359999736150107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,4096,32,0.003458133339881897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,4096,32,0.015406933426856995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,2048,0.009895466764767965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,65536,0.1771722634633382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,65536,0.1277130683263143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,51200,0.1412000020345052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,1536,0.00830080012480418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,51200,0.1038581371307373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,16384,0.05198186635971069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,16384,0.0421674648920695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,1024,0.0068351998925209045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,12288,0.039692799250284835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,12288,0.03710720141728719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,768,0.006425599753856659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,10240,0.030393600463867188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,10240,0.03394453525543213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,512,0.005994666616121928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,8192,0.02490773399670919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,8192,0.0286463995774587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,8192,0.021915733814239502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,7168,0.023982934157053628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,256,0.005453866720199585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,7168,0.027153066794077557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,6144,0.02123519976933797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,4096,128,0.005155199766159057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,6144,0.02540160020192464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,5120,0.018105600277582803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,5120,0.023297067483266196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,65536,0.14500160217285157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,4096,0.014652799566586813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,4096,0.021658666928609214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,51200,0.11466240088144938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,3584,0.013333333532015481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,3584,0.020889600118001304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,16384,0.03948800166447957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,3072,0.011923199892044068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,3072,0.020457599560419717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,12288,0.030565333366394044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,2560,0.010121599833170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,2560,0.019471999009450278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,10240,0.026392533381779985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,2048,0.008839466174443563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,2048,0.01780160069465637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,7168,0.020614399512608846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,1536,0.0075082664688428235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,1536,0.016458666324615477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,6144,0.01845653255780538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,5120,0.015204266707102457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,1024,0.005373866856098175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,1024,0.016189866264661155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,4096,0.013602133591969809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,768,0.004841599861780802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,768,0.015990400314331056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,768,0.006202666461467743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,512,0.004154666761557261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,512,0.015591466426849365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,3584,0.012552533547083536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,256,0.0036906667053699495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,256,0.015128533045450846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,3072,0.01106666624546051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,128,0.0033471999069054925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,2560,0.010757333040237427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,128,0.015440000096956888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,64,0.0031744000812371576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,2048,0.009177600344022114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,64,0.015129599968592325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3584,32,0.003307733436425527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3584,32,0.015161599715550741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,65536,0.15157225926717122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,65536,0.11391999721527099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,51200,0.12022613684336345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,51200,0.09358506997426351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,51200,0.12050027052561443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,1536,0.00782719999551773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,16384,0.04328213135401408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,16384,0.03964373270670573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,1024,0.006841599941253662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,12288,0.03505706787109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,12288,0.03359573284784953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,10240,0.030096000432968138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,10240,0.03312000036239624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,8192,0.022397865851720176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,8192,0.02967573404312134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,512,0.005881600081920624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,7168,0.020482132832209267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,7168,0.02707093358039856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,256,0.005454933146635691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,6144,0.01804479956626892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,6144,0.02527359922726949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3584,128,0.005206400156021118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,5120,0.01818880041440328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,5120,0.02257386644681295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,65536,0.1530303955078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,4096,0.015316266814867655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,4096,0.021331199010213218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,16384,0.04057600100835164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,12288,0.03179093400637309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,3584,0.013498666882514953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,10240,0.027139200766881304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,3584,0.020478934049606323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,3072,0.012105600039164225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,3072,0.02000853419303894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,2560,0.009428266684214275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,2560,0.019024000565210978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,8192,0.022733867168426514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,2048,0.008200533191363017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,2048,0.01781546672185262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,2048,0.008881066242853801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,1536,0.007170133292675018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,1536,0.016577066977818807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,7168,0.020808533827463786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,1024,0.005743999779224396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,1024,0.0160778671503067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,6144,0.018351999918619792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,768,0.00494400014479955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,768,0.01602240006128947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,5120,0.015589333573977151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,512,0.0041184000670909885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,512,0.015804800391197204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,4096,0.01360640029112498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,256,0.0036618667344252265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,256,0.015309866269429526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,3584,0.01279253363609314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,3072,0.011448533336321513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,128,0.003389866650104523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,128,0.015131733814875283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,64,0.003083733220895131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,64,0.015069866180419922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,2560,0.010426666339238484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,3072,32,0.003193599979082743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,3072,32,0.015015467007954916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,65536,0.130021333694458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,65536,0.10282453695933025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,1536,0.007820799946784973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,51200,0.10314133167266845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,51200,0.08502186934153239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,16384,0.03784960110982259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,1024,0.007181866466999054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,16384,0.0379093329111735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,12288,0.030453334252039593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,16384,0.03778453270594279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,12288,0.030920533339182536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,10240,0.026964267094930012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,10240,0.027464532852172853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,768,0.006433066725730896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,8192,0.0229418675104777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,8192,0.02473919987678528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,512,0.005841066439946493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,256,0.0052597333987553915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,7168,0.018377600113550822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,7168,0.023721599578857423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,3072,128,0.005093333125114441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,6144,0.016081066926320393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,6144,0.022705066204071044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,65536,0.13584532737731933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,5120,0.013387733697891235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,5120,0.02191466689109802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,51200,0.10547093550364177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,4096,0.011366400122642516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,12288,0.029173332452774047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,4096,0.02025493383407593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,4096,0.013097600142161051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,3584,0.010595200459162395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,3584,0.02000746726989746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,3072,0.009693866968154908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,3072,0.019382399320602418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,10240,0.02505386670430501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,2560,0.008500267068545024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,2560,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,2048,0.007461333274841308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,2048,0.01685973405838013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,8192,0.021051732699076335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,1536,0.006326400240262349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,1536,0.016697599490483602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,7168,0.019565866390864054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,1024,0.0049898669123649595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,1024,0.016034133235613503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,6144,0.017356799046198527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,768,0.004444799820582072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,768,0.01569066643714905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,5120,0.014802133043607077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,512,0.003921066721280416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,512,0.015773866573969522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,3584,0.01225386659304301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,256,0.003390933324893316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,256,0.014967466394106546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,3072,0.010657067100207012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,128,0.003125333289305369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,128,0.014691199858983359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,2560,0.009252267082532246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,64,0.0030026666820049284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,64,0.01460906664530436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2560,32,0.003058133274316788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2560,32,0.014629333217938741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,65536,0.10989226500193279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,65536,0.08989013036092122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,2048,0.008297599852085114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,51200,0.08675413131713867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,1536,0.007292800148328145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,51200,0.07354453404744467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,16384,0.03174933393796285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,16384,0.033659732341766356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,1024,0.0068693334857622785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,12288,0.025769599278767902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,12288,0.029203200340270997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,768,0.006146133442719777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,10240,0.021652267376581828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,512,0.0056309332450230915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,256,0.005381333331267038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,8192,0.018080000082651773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2560,128,0.005110399921735128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,8192,0.024937599897384644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,7168,0.017214934031168617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,7168,0.023838933308919272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,10240,0.02759466568628947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,6144,0.016619732975959776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,65536,0.13574186960856122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,6144,0.02230186661084493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,5120,0.014276267091433207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,5120,0.02096959948539734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,51200,0.10392746925354004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,4096,0.012174933155377706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,16384,0.034825599193573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,12288,0.0271509329477946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,3584,0.01046720047791799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,3584,0.018901334206263224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,10240,0.023542400201161703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,3072,0.009453866879145305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,3072,0.017909334103266398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,4096,0.019870932896931967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,2560,0.007849599917729695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,2560,0.017589332660039265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,8192,0.01986560026804606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,2048,0.006970666845639547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,2048,0.016938666502634682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,7168,0.01873813271522522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,1536,0.0059008002281188965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,1536,0.01648853321870168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,6144,0.01689066688219706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,1024,0.004488533238569895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,1024,0.015548800428708395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,5120,0.014019200205802917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,768,0.004149333387613296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,4096,0.012170666456222534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,768,0.015974400440851848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,512,0.0037546666959921518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,512,0.014936533570289613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,3584,0.011361066500345867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,3072,0.009820800026257832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,256,0.003271466741959254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,256,0.014630400141080222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,2560,0.008973866701126099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,128,0.0031850665807724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,128,0.014793599645296732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,2048,0.008022400240103405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,64,0.003047466774781545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,64,0.014889599879582724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,2048,32,0.0030741333961486817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,1536,0.007559466858704884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,2048,32,0.014759467045466105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,65536,0.08577067057291667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,65536,0.07680213451385498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,51200,0.06637866497039795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,51200,0.06435946623484293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,1024,0.006524799764156342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,16384,0.02513386607170105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,16384,0.03067306677500407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,16384,0.030557866891225176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,768,0.006209066510200501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,12288,0.01994026700655619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,12288,0.027986133098602296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,10240,0.017090133825937905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,10240,0.026680533091227216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,512,0.005770666897296906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,8192,0.015642666816711427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,8192,0.024734934171040855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,7168,0.015348266561826071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,7168,0.022872533400853476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,7168,0.017131733894348144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,6144,0.013730133573214212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,6144,0.022156800826390585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,256,0.0052597333987553915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,5120,0.01236799955368042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,5120,0.02087786595026652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,2048,128,0.0048885335524876915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,4096,0.010359467069307963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,4096,0.019425066312154134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,65536,0.11206293106079102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,3584,0.009553066889444987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,3584,0.018373332420984902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,51200,0.08910933335622152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,3072,0.008780800302823384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,3072,0.017786665757497152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,12288,0.023975465695063272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,2560,0.008406399687131246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,2560,0.017667200167973837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,10240,0.02142826716105143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,2048,0.006557866434256236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,2048,0.01715839902559916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,8192,0.018463999032974243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,1536,0.005752533177534739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,1536,0.01634773313999176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,5120,0.012732799847920737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,6144,0.01544426679611206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,1024,0.004714666803677877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,1024,0.01583999991416931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,768,0.004344533383846283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,768,0.01552959978580475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,4096,0.010699733098347982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,512,0.003751466671625773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,512,0.015597866972287497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,3584,0.010205866893132527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,256,0.0033514666060606635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,3072,0.008810666203498841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,256,0.014979199568430582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,128,0.0030794667700926462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,128,0.01476693352063497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,2560,0.008306133250395458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,64,0.002937600016593933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,2048,0.007534933090209961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,64,0.014827733238538107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1536,32,0.0030239999294281008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1536,32,0.014747732877731323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,65536,0.05772373278935751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,65536,0.06567146778106689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,65536,0.10453973611195881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,51200,0.04690879980723063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,51200,0.053990399837493895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,51200,0.07549440066019694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,16384,0.01811199982961019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,16384,0.026951466004053754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,12288,0.013923199971516928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,12288,0.027450666824976606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,1024,0.006413866579532623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,10240,0.012530133128166199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,10240,0.024936532974243163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,10240,0.019008000691731773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,8192,0.013489066561063131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,768,0.0056970665852228795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,8192,0.016391467054684958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,7168,0.013804800311724343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,7168,0.022106667359670006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,7168,0.015293866395950317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,512,0.005305600166320801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,6144,0.012555733323097229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,6144,0.013097600142161051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,6144,0.021381332476933797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,1536,0.007089066505432129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,5120,0.020026665925979615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,5120,0.011060266693433126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,4096,0.009430399537086487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,4096,0.018981333573659262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,256,0.005340800185998281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,3584,0.008995200196901958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1536,128,0.004823466638724009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,3584,0.009525332848230999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,3072,0.008203733464082081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,3072,0.01815999945004781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,8192,0.02315733234087626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,3072,0.008472533027331034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,2560,0.0072543998559316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,2560,0.017435733477274576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,2048,0.006308266520500183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,2048,0.01694186727205912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,2048,0.007366399963696797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,1536,0.005297066768010458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,1536,0.016597333550453185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,1536,0.006776533524195353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,1024,0.004446933170159658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,1024,0.015681067109107973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,1024,0.0059914668401082356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,768,0.004108799993991852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,16384,0.02866026759147644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,768,0.016124799847602844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,512,0.0036309334139029183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,768,0.005679999788602194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,512,0.015221333503723145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,256,0.003221333275238673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,256,0.014973866939544677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,12288,0.021090133984883627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,256,0.004910933474699656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,128,0.0030229332546393077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,128,0.015473066767056783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,64,0.0028213332096735638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,64,0.014907733599344889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,32,0.0028533334533373516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,32,0.014843733112017313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,65536,0.04670826594034831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,65536,0.05873173475265503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,1024,5120,0.010908800363540649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,51200,0.038286932309468585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,51200,0.04923093318939209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,51200,0.07649706999460856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,16384,0.014075733224550881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,16384,0.026855466763178508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,16384,0.026946133375167845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,12288,0.014865066607793173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,1024,3584,0.018889600038528444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,12288,0.021307732661565146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,12288,0.023541333278020223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,10240,0.011427199840545655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,8192,0.012135466933250428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,4096,0.009796266754468281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,10240,0.022882133722305298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,8192,0.020408533016840615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,7168,0.01990613341331482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,10240,0.01814613342285156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,512,0.005338666836420695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,6144,0.010502400000890096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,7168,0.011057066917419433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,6144,0.020958934227625528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,6144,0.012442666292190551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,5120,0.00955839951833089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,5120,0.019939200083414713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,5120,0.011194666226704914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,4096,0.00969599982102712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,4096,0.01883413394292196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,3584,0.00886079967021942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,3584,0.018667733669281004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,3072,0.007968000074227651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,3072,0.017992534240086875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,3072,0.00843519965807597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,2560,0.008072533210118612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,2560,0.0075914666056633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,2560,0.008044800162315369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,2560,0.017052799463272095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,2048,0.006385066608587901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,2048,0.016757333278656007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,1536,0.005635199944178263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,1536,0.016053332885106405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,65536,0.09805973370869955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,1024,0.0043722664316495265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,1024,0.015629866719245912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,4096,0.008050133287906647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,768,0.004008533308903376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,1024,128,0.0048213332891464235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,768,0.016260266304016113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,8192,0.015099733074506124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,512,0.003701333453257879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,768,0.005434666574001312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,256,0.0032511999209721885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,512,0.005167999863624572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,256,0.0049002667268117275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,256,0.014710399508476257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,128,0.003019733230272929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,128,0.014643200238545737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,64,0.002794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,7168,0.01381119986375173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,64,0.0145578662554423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,768,32,0.0029290666182835894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,32,0.014490666985511779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,768,512,0.014819199840227762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,65536,0.033562668164571124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,65536,0.09290026823679606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,65536,0.05210453271865845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,51200,0.027641600370407103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,51200,0.04513066609700521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,51200,0.07138880093892416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,3584,0.009383466839790345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,16384,0.013150933384895324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,16384,0.024272000789642333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,12288,0.009970133503278095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,12288,0.021970132986704506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,12288,0.020374399423599244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,10240,0.010986666878064473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,2048,0.007201066613197327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,8192,0.009731200337409974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,10240,0.017407999436060587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,1536,0.0066431999206542965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,8192,0.020678400993347168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,7168,0.009037866195042928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,7168,0.019731199741363524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,7168,0.013525333007176718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,6144,0.008057599763075511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,6144,0.012350933750470479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,6144,0.01924906571706136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,1024,0.005915733178456625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,5120,0.01954453388849894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,4096,0.00754559983809789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,4096,0.01821546753247579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,4096,0.009646933277448018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,3584,0.0070816000302632645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,3584,0.018153599898020425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,16384,0.026570665836334228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,3072,0.006596266726652781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,768,128,0.0047423998514811196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,10240,0.020383999745051066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,3072,0.01751040021578471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,2560,0.0062720000743865965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,2560,0.017069866259892784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,2560,0.007964799801508587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,2048,0.005566933254400889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,2048,0.016378666957219443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,1536,0.01600320041179657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,5120,0.0074890668193499255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,1536,0.006613333523273468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,1024,0.004198400179545084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,5120,0.01107413371404012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,1024,0.005755733450253805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,1024,0.015748266379038492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,768,0.0037952000896135964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,512,0.0036138666172822317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,768,0.005354666709899902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,512,0.005209599932034811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,512,0.01530880033969879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,256,0.0032586666444937387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,256,0.015063466628392539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,256,0.004829866687456766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,128,0.0029696000119050344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,128,0.0046069333950678505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,64,0.0028714666763941447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,128,0.014656000336011252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,32,0.0031061333914597826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,64,0.014916266997655234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,32,0.014845866958300272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,65536,0.023808000485102336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,65536,0.0882805347442627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,65536,0.04508800109227498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,51200,0.018606932957967122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,51200,0.03571946620941162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,51200,0.06938453515370688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,16384,0.009641599655151368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,16384,0.021943465868632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,16384,0.025054933627446492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,12288,0.009293867150942485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,512,1536,0.004850133260091146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,12288,0.020381865898768108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,10240,0.008451199531555176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,12288,0.019782400131225585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,8192,0.014904533823331198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,8192,0.007209600011507671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,10240,0.017121066649754844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,8192,0.019362133741378785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,7168,0.006958933174610138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,8192,0.014470400412877402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,7168,0.020974934101104736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,6144,0.006457599997520447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,7168,0.0135861337184906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,512,768,0.01572160025437673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,5120,0.0075328002373377485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,6144,0.01999146739641825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,5120,0.018959999084472656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,5120,0.010982400178909302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,4096,0.006278400123119354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,3584,0.007518933216730754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,4096,0.009684266646703084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,4096,0.018230400482813516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,3584,0.017899733781814576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,3584,0.009269332885742188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,3072,0.006785066425800323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,3072,0.017450666427612303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,2560,0.006039466460545858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,2560,0.016804265975952148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,2560,0.007995733122030894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,2048,0.005387733379999796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,2048,0.01640959978103638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,2048,0.007066666583220164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,1536,0.004673066735267639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,1536,0.015941333770751954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,1536,0.00661653329928716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,1024,0.004098133246103922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,3072,0.008311466872692108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,1024,0.01544533371925354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,1024,0.005809066692988077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,768,0.0036490666369597114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,10240,0.021313067277272543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,768,0.005295999844868978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,512,0.003437866767247518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,768,0.01562666694323222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,2048,0.0071829333901405334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,256,0.003115733216206233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,512,0.014978133638699851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,6144,0.012160000205039979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,128,0.0028149334092934927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,256,0.01477226714293162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,128,0.004748799900213877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,128,0.01452906628449758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,32,0.00275093341867129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,64,0.01432960033416748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,256,32,0.01455466647942861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,65536,0.014269866545995078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,65536,0.038021334012349445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,65536,0.08950933615366617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,51200,0.011239467064539592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,512,3584,0.009340799848238627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,51200,0.03314346671104431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,16384,0.007964799801508587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,16384,0.0206058661142985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,12288,0.007335466643174489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,16384,0.0251093327999115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,12288,0.020869332551956176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,10240,0.006846933563550313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,10240,0.019740800062815346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,10240,0.01721386710802714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,8192,0.006486399968465169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,8192,0.019356799125671387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,8192,0.014549332857131957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,7168,0.006274133423964183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,512,0.00517546683549881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,7168,0.01930239995320638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,256,0.00484799991051356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,6144,0.006056533257166544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,7168,0.013500799735387167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,256,64,0.0026869334280490874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,5120,0.0065642664829889934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,6144,0.019074134031931558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,5120,0.010961066683133442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,5120,0.019285333156585694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,4096,0.006095999975999197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,3584,0.006548266609509785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,4096,0.009428266684214275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,4096,0.018461867173512777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,3584,0.009202133615811665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,3072,0.006196266909440359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,3584,0.017949867248535156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,2560,0.006120533247788747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,3072,0.008312533299128216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,3072,0.01781546672185262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,2560,0.007859200239181519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,2048,0.005433600147565206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,2560,0.01691733400026957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,51200,0.07143466472625733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,1536,0.004786133269468943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,2048,0.007085866729418437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,2048,0.01644480029741923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,1024,0.004062933226426443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,1536,0.006702933212121327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,1536,0.016269866625467935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,1024,0.005761066575845083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,1024,0.015610667069753012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,512,0.0034453332424163817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,768,0.015332266688346863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,768,0.0054730668663978575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,256,0.0030613332986831666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,512,0.005207466582457224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,512,0.015355733036994935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,256,0.01469013293584188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,128,0.002820266783237457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,64,0.002705066651105881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,128,0.01460906664530436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,32,0.00276053324341774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,64,0.014587733149528503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,128,32,0.014578133821487427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,65536,0.010351999600728353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,51200,0.010574932893117268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,65536,0.03560959895451864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,16384,0.00678719977537791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,51200,0.030110933383305866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,16384,0.020925867557525634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,12288,0.006229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,12288,0.019564799467722573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,10240,0.0067007998625437425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,10240,0.019663999478022255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,8192,0.006345599889755249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,8192,0.01913706660270691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,7168,0.006077866752942403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,7168,0.019296000401178993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,6144,0.005947733422120413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,128,768,0.003688533355792364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,6144,0.018687999248504637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,5120,0.006371200084686279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,4096,0.00598826656738917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,5120,0.01871466636657715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,3584,0.006375466783841451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,4096,0.01830293337504069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,128,0.004786133269468943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,3072,0.0060810665289560955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,3584,0.01766506632169088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,2560,0.006020266811052958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,3072,0.01735573410987854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,2048,0.005331199864546458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,2560,0.016820265849431356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,1536,0.004702933132648468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,2048,0.01628373364607493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,1024,0.004071466624736786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,1536,0.015811199943224587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,768,0.003730133424202601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,1024,0.015732266505559287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,512,0.0033439998825391137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,768,0.015154133240381876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,256,0.0029813334345817565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,128,0.002810666710138321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,256,0.014734933773676554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,64,0.002682666728893916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,128,0.014725333452224732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,64,32,0.00266239990790685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,64,0.014457600315411887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,32,0.01464959979057312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,65536,0.0120469331741333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,51200,0.008784000078837078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,65536,0.0379253347714742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,51200,0.030454399188359578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,16384,0.0067893331249554946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,12288,0.006237866481145223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,16384,0.020881066719690956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,256,3072,0.008390399813652038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,12288,0.019733333587646486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,8192,0.00639466643333435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,10240,0.019792000452677407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,6144,0.01225920021533966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,7168,0.006077866752942403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,8192,0.01885333259900411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,6144,0.005942399799823761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,7168,0.018918399016062418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,5120,0.00631466656923294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,6144,0.018742400407791137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,4096,0.00589333325624466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,5120,0.019006933768590292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,3584,0.006359466910362243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,4096,0.018145066499710084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,3072,0.0060149331887563075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,3584,0.017998933792114258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,3072,0.017356799046198527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,2560,0.006076799829800924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,2048,0.005355733136336008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,2560,0.016734933853149413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,2048,0.016345600287119545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,1024,0.004013866682847341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,1536,0.01571626663208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,1024,0.015531733632087708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,768,0.003605333218971888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,512,0.0033834666013717652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,768,0.01530346671740214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,256,0.003068800022204717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,512,0.014889599879582724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,128,0.0028373333315054577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,256,0.014632532993952433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,64,0.00277866671482722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,128,0.0144405335187912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,32,0.0026911998788515727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,64,0.014417066176732381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,32,32,0.01439466675122579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,10240,0.006764799853165944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,12288,0.019819732507069907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,113,32,1536,0.004761599997679392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,113,64,512,0.014872533082962037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,113,128,256,0.004818133513132731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,16384,0.4069013277689616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,16384,0.37521492640177406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,12288,0.5810325622558594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,12288,0.31292692820231116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,12288,0.28423681259155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,51200,1.2208564758300782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,10240,0.4942218780517578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,51200,1.1318655649820963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,10240,0.26384639739990234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,8192,0.38934720357259117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,8192,0.246780792872111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,8192,0.20960426330566406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,16384,0.7635658899943034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,7168,0.1908394654591878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,51200,2.380149332682292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,7168,0.3980287869771322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,7168,0.17226559321085613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,6144,0.14933973948160809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,6144,0.29656747182210286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,6144,0.18997972806294758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,5120,0.12744106451670328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,5120,0.1445504029591878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,4096,0.20042986869812013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,4096,0.11919466654459636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,5120,0.2860543886820475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,3584,0.10777706305185955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,3584,0.1764479955037435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,3072,0.15317440032958984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,3072,0.10496853192647297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,3072,0.08334506352742513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,2560,0.12878613471984862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,2560,0.08295893669128418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,2048,0.10542933146158855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,2048,0.07045973141988118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,2560,0.07859520117441812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,1536,0.08139626979827881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,1536,0.058208000659942624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,1024,0.05728533267974854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,1024,0.045525332291920975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,1024,0.035547733306884766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,768,0.04533226490020752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,768,0.039290666580200195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,768,0.03158613244692485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,512,0.03325973351796468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,512,0.031162667274475097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,512,0.02435200015703837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,256,0.018915200233459474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,256,0.018380800882975258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,256,0.024099200963974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,128,0.011262933413187664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,128,0.02143893241882324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,128,0.016217600305875143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,64,0.008756267031033833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,65536,32,0.008189866443475087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,64,0.019707733392715455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,65536,32,0.020601600408554077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,10240,0.2381429354349772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,1536,0.052008533477783205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,4096,0.10450666745503742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,3584,0.09437867005666098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,65536,1.2343690236409506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,65536,2048,0.05871786673863729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,51200,0.9736170450846353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,16384,0.60011412302653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,16384,0.3751605351765951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,65536,2.3823776245117188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,12288,0.4898197491963704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,51200,1.905409113566081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,12288,0.24990827242533364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,10240,0.22851306597391763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,8192,0.3057098706563314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,10240,0.44800214767456054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,8192,0.17489813168843588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,7168,0.3116544087727865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,7168,0.15506134033203126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,7168,0.15712213516235352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,6144,0.232093874613444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,6144,0.13609174092610676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,5120,0.19330453872680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,5120,0.1168405294418335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,65536,1.1793898264567058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,16384,0.301798407236735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,4096,0.1850389321645101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,4096,0.10068693161010742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,51200,0.9100479761759439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,3584,0.13860267003377277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,12288,0.23155946731567384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,3584,0.09902613162994385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,3072,0.13621120452880858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,10240,0.22072106997172036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,3072,0.08067733446756999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,8192,0.1637898604075114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,2560,0.11667946974436443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,2560,0.07025919755299886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,2048,0.05999360084533691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,2048,0.09574293295542399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,1536,0.0733845313390096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,1536,0.05032853285471598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,6144,0.12028373082478841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,1024,0.05115093390146891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,1024,0.04009706576665242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,5120,0.10168533325195313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,768,0.03642880121866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,768,0.03619626760482788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,4096,0.08457600275675456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,512,0.029689600070317585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,512,0.02834133307139079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,3584,0.0748650630315145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,256,0.015415466825167336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,256,0.02322346568107605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,3072,0.06592533191045126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,2560,0.05721919933954874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,128,0.010963199536005656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,128,0.020693333943684895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,2048,0.047141333421071366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,64,0.008342400193214417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,64,0.019873066743214925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,51200,32,0.007796266674995422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,1536,0.038252798716227214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,51200,32,0.019973333676656088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,1024,0.02940586606661479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,65536,0.4047840118408203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,65536,0.7784287770589192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,768,0.02523840069770813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,51200,0.5725525538126628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,51200,0.3241408030192057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,51200,0.2970325469970703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,16384,0.2232682704925537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,512,0.01918720006942749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,16384,0.13125332991282146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,12288,0.14378132820129394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,16384,0.1033087968826294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,12288,0.07898666858673095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,12288,0.0921130657196045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,256,0.014782933394114175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,10240,0.06677546501159667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,10240,0.12088960011800129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,10240,0.07970986366271973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,8192,0.06680106321970622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,8192,0.09793173472086589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,8192,0.05531626542409261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,7168,0.0860853354136149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,65536,0.37757867177327475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,5120,0.06346133152643839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,6144,0.055118934313456214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,7168,0.0500053326288859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,6144,0.07554453214009603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,4096,0.05212053457895914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,4096,0.032232532898585006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,5120,0.03814186652501424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,4096,0.04215253194173177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,3584,0.046029865741729736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,51200,128,0.012401066223780314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,5120,0.048802133401234946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,3584,0.02932479977607727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,3072,0.040700801213582355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,3072,0.0262442668279012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,3072,0.036150399843851724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,2560,0.03468586603800456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,2560,0.03295893271764119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,2048,0.028862933317820232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,2560,0.023191465934117635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,2048,0.028540800015131634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,1536,0.02298240065574646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,2048,0.019817600647608437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,1024,0.01596799989541372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,1536,0.016200533509254454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,1536,0.024939733743667602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,768,0.012518399953842163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,1024,0.012714667121569314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,1024,0.02222613294919332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,768,0.020297600825627645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,512,0.00942080020904541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,256,0.006615466872851054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,512,0.009955199559529622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,512,0.018744534254074095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,128,0.004436266422271728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,256,0.015848533312479655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,64,0.00413226659099261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,128,0.007123200098673503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,16384,32,0.004497066636880239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,64,0.01616213321685791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,32,0.016416000326474507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,7168,0.06130880117416382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,6144,0.043958401679992674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,65536,0.33172906239827477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,3584,0.03920213381449382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,65536,0.5772160212198894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,16384,0.18341013590494792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,256,0.007558399935563405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,16384,128,0.015974400440851848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,51200,0.23070613543192545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,65536,0.2936373392740885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,51200,0.26741228103637693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,16384,0.09140053590138754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,12288,0.10843199888865154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,16384,0.08103253046671549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,12288,0.07758613427480063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,10240,0.09164693355560302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,12288,0.06511146624883016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,10240,0.06387946605682374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,8192,0.07386879920959473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,10240,0.05289066632588705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,8192,0.05488959948221842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,6144,0.05720426638921102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,7168,0.03951679865519206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,6144,0.04571199814478556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,5120,0.04863786697387695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,6144,0.034835199515024826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,5120,0.040725334485371904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,5120,0.03031040032704671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,4096,0.03994133472442627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,4096,0.025510400533676147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,4096,0.035689600308736164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,3584,0.035366400082906084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,3584,0.033446399370829265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,3072,0.031430399417877196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,3072,0.030796800057093305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,51200,0.45590505599975584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,16384,768,0.011522133151690166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,2560,0.030331732829411824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,2560,0.027698133389155073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,2048,0.022431999444961548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,2048,0.024654932816823325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,2048,0.01525759994983673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,7168,0.06502399841944376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,1536,0.01760639945665995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,1536,0.012885333100954691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,7168,0.04995413223902385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,1024,0.01209920048713684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,1024,0.020414932568868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,768,0.010065066814422607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,768,0.019463467597961425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,768,0.009622400005658466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,512,0.007897600034872691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,512,0.016821332772572837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,512,0.007421866556008657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,256,0.0046517332394917805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,256,0.015706666310628257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,256,0.006503466765085857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,128,0.003985066711902618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,128,0.015521066387494406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,128,0.006074666480223338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,64,0.003692800054947535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,64,0.015763200322786965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,12288,32,0.004043733328580856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,32,0.01581333378950755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,12288,1536,0.022318933407465616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,8192,0.04389653205871582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,65536,0.49811627070109055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,65536,0.29165865580240885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,3584,0.023332266012827556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,51200,0.39379841486612954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,16384,0.12012159824371338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,51200,0.2336341381072998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,3072,0.020683733622233073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,16384,0.06952319939931234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,12288,0.09284053643544515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,12288,0.06548053423563639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,16384,0.0904917319615682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,10240,0.07769813537597656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,10240,0.05786986748377482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,2560,0.01817173361778259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,8192,0.0627946654955546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,8192,0.04918933312098185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,7168,0.055454933643341066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,7168,0.04519466559092204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,7168,0.034024532636006674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,6144,0.049277865886688234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,12288,1024,0.010754133264223736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,6144,0.04084159930547078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,6144,0.03016746640205383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,5120,0.04164479970932007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,5120,0.03699093262354533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,4096,0.03420159816741943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,5120,0.026125866174697875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,4096,0.032509867350260416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,3584,0.030433066685994464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,3584,0.03032853404680888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,3584,0.02030613422393799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,3072,0.026877866188685103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,3072,0.02797120014826457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,2560,0.023370667298634847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,2560,0.025546665986378985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,2560,0.0156960000594457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,2048,0.019299199183781944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,2048,0.023859200874964397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,2048,0.013693867127100625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,1536,0.015105066696802774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,1536,0.02151040037473043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,65536,0.25031894048055015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,1024,0.01207360029220581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,1024,0.020267732938130698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,1024,0.009699199597040813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,51200,0.19820480346679686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,768,0.009078400333722432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,768,0.01918399930000305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,512,0.007032533486684163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,12288,0.05348480145136515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,512,0.016497066617012023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,512,0.006758399804433187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,256,0.004640000065167745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,8192,0.03780906597773234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,256,0.015848533312479655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,10240,0.04558080037434896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,128,0.004036266605059306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,128,0.005579733351866404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,128,0.015365333358446757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,64,0.0037237333754698435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,10240,32,0.0038773333032925926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,64,0.015705600380897522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,10240,32,0.015631999572118124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,4096,0.02223466634750366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,65536,0.23314132690429687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,3072,0.017749333381652833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,65536,0.41408958435058596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,51200,0.3014165242513021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,51200,0.18780694007873536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,51200,0.16935680707295736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,16384,0.10648746490478515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,1536,0.011638399958610535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,256,0.005931733548641205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,12288,0.07347520192464194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,12288,0.05552639961242676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,10240,0.06185386578241984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,10240,0.04914666811625163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,8192,0.050563200314839685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,8192,0.04268053372701009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,10240,768,0.008281599978605907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,16384,0.06781653563181558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,7168,0.044512001673380534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,7168,0.03915199836095174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,6144,0.039511466026306154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,6144,0.0360426664352417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,5120,0.03339093526204427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,5120,0.032382933298746745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,5120,0.022344533602396646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,4096,0.027499733368555705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,4096,0.028714666763941448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,65536,0.21244373321533203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,3584,0.027985066175460815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,3584,0.02656960090001424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,16384,0.059455998738606776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,3072,0.02190293272336324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,12288,0.0451039989789327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,3072,0.024919466177622477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,3072,0.015494400262832641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,2560,0.018986666202545167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,2560,0.02327359914779663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,10240,0.03841813405354817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,2048,0.01543786625067393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,2048,0.021600000063578286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,1536,0.012113066514333089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,8192,0.03227413296699524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,1536,0.020106667280197145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,1024,0.009112532933553059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,1024,0.018611200650533042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,7168,0.028835199276606244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,768,0.007812266548474629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,768,0.016577066977818807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,6144,0.025430399179458617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,512,0.006178133189678192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,512,0.01601920028527578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,4096,0.019048533837000527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,256,0.004347733159859975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,256,0.015844266613324484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,3584,0.01748266617457072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,128,0.003700266778469086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,128,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,128,0.005875200033187866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,64,0.003470933437347412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,64,0.015576533476511636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,8192,32,0.0039061332742373147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,8192,32,0.015677866339683533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,2560,0.01411733329296112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,2048,0.01241386632124583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,1536,0.01090773344039917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,65536,0.22236480712890624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,1024,0.009064533313115438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,51200,0.1683626651763916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,51200,0.28215039571126305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,16384,0.09366933504740396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,16384,0.06265706618626912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,12288,0.06493653456370035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,12288,0.051438931624094644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,12288,0.041510399182637533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,768,0.007667199770609538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,10240,0.05515733162562052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,10240,0.045398398240407305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,512,0.00681279997030894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,8192,0.04433066844940185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,8192,0.029371732473373414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,8192,0.039230934778849286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,65536,0.35497493743896485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,7168,0.03954773346583049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,7168,0.036489601929982504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,6144,0.03512426614761353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,7168,0.02940906683603923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,6144,0.033405868212382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,5120,0.03387093146642049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,5120,0.030423466364542646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,4096,0.024637866020202636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,8192,256,0.006289066871007283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,4096,0.02654293378194173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,3584,0.022039467096328737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,3584,0.02505279978116353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,3584,0.015430399775505066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,3072,0.019633066654205323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,3072,0.023833600680033366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,65536,0.20057493845621743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,51200,0.15778560638427735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,2560,0.016552533706029257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,2560,0.02246399919191996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,2560,0.012938666343688964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,2048,0.013716266552607218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,16384,0.05354559818903605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,2048,0.020875734090805054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,1536,0.011000532905260723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,1536,0.01949119965235392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,10240,0.035595734914143876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,1024,0.008529067039489746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,1024,0.016793600718180337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,6144,0.023465599616368613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,768,0.007315200070540111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,5120,0.02042986750602722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,768,0.015890133380889893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,512,0.004911999901135763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,768,0.007102933526039123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,4096,0.017314134041468303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,512,0.016124799847602844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,256,0.0041184000670909885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,256,0.015434666474660238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,256,0.00580266664425532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,128,0.0034965333839257562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,128,0.015304533640543619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,64,0.0034965333839257562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,3072,0.01413653294245402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,64,0.015453867117563882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,7168,32,0.0036576000352700555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,7168,32,0.015196800231933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,65536,0.19026026725769044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,65536,0.2994442621866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,65536,0.18187626202901203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,51200,0.2387925306955973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,51200,0.15404693285624188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,16384,0.0853546698888143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,16384,0.05649493137995402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,12288,0.05662399927775065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,12288,0.04713706572850545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,1536,0.00993173321088155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,2048,0.011370666821797689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,10240,0.04822719891866048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,10240,0.0421887993812561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,8192,0.03922986586888631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,8192,0.03649066686630249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,1024,0.007796266674995422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,7168,0.03471786578496297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,7168,0.03371200164159139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,512,0.006212266782919565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,6144,0.03096853295962016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,6144,0.03102933367093404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,7168,128,0.005419733126958212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,5120,0.026372265815734864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,5120,0.02757226626078288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,51200,0.14663999875386555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,4096,0.024471465746561685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,4096,0.02549546758333842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,16384,0.04896320104598999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,3584,0.019581866264343262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,3584,0.02395306626955668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,12288,0.03750933408737182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,3072,0.017373865842819212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,10240,0.03215679923693339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,3072,0.022663466135660806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,2560,0.014622933665911355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,2560,0.021661865711212158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,8192,0.02689066727956136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,2048,0.012385066350301106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,2048,0.020390399297078452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,7168,0.024117332696914674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,1536,0.009925333658854167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,1536,0.019037866592407228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,6144,0.021243733167648316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,1024,0.00781333347161611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,1024,0.01651306649049123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,5120,0.018523732821146645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,768,0.006885333359241486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,768,0.016294399897257485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,4096,0.015405866503715514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,512,0.0047541335225105286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,512,0.01648853321870168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,3584,0.014409599701563516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,256,0.003973333289225897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,256,0.015318399667739869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,3072,0.012941867113113403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,128,0.0034293333689371743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,128,0.014913066228230795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,2560,0.011935999989509583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,64,0.00329066663980484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,64,0.015130666891733804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,6144,32,0.00344106654326121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,6144,32,0.015107199549674988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,2048,0.010487467050552368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,1536,0.009150933225949604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,65536,0.24583679835001626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,65536,0.16285120646158854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,51200,0.1941215991973877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,51200,0.13210559686024984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,16384,0.0713759978612264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,1024,0.0072405333320299785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,16384,0.05191786686579386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,768,0.0067104001839955645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,12288,0.05632640123367309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,12288,0.0433898647626241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,512,0.006151466568311056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,10240,0.04925440152486165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,10240,0.03865706523259481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,256,0.005682133138179779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,10240,0.029632000128428142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,8192,0.033480532964070636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,8192,0.03308479984601338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,6144,128,0.005235200126965841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,7168,0.030538666248321533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,7168,0.031066666046778362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,6144,0.027020800113677978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,6144,0.02837439974149068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,65536,0.17816106478373211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,5120,0.025655466318130492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,5120,0.025718400875727337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,51200,0.13653225898742677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,16384,0.044351998964945474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,4096,0.02367146611213684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,3584,0.017108267545700072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,3584,0.022612265745798745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,12288,0.03500586748123169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,3072,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,3072,0.021833600600560506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,4096,0.021347200870513915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,2560,0.012819199760754903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,4096,0.014471466342608133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,8192,0.024445867538452147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,2560,0.020614399512608846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,2048,0.010800000031789143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,2048,0.01953173279762268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,7168,0.022422399123509726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,1536,0.00897920032342275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,1536,0.018101332585016887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,1024,0.007117866476376851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,6144,0.019833600521087645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,1024,0.016595199704170227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,768,0.0059456000725428265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,768,0.016037333011627197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,768,0.006656000018119812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,512,0.004279466470082601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,512,0.015864533185958863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,5120,0.017126399278640746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,256,0.003773866593837738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,256,0.015129599968592325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,1024,0.007065600156784058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,128,0.0033781332274278007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,3584,0.013558399677276612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,128,0.015014400084813436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,128,0.005336533486843109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,64,0.0031114667654037476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,5120,32,0.0032992000381151833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,64,0.015046399831771851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,5120,32,0.015015467007954916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,3072,0.012076800068219502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,65536,0.20159786542256675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,65536,0.13981547355651855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,51200,0.1134442647298177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,51200,0.15970880190531414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,16384,0.06285226742426554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,16384,0.0481600006421407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,16384,0.04034666617711385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,12288,0.04416319926579793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,12288,0.037646933396657305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,2048,0.010161067048708599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,2560,0.0113237331310908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,10240,0.03827519814173381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,10240,0.034140801429748534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,1536,0.008763733506202697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,8192,0.032202666997909545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,8192,0.029563732941945392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,8192,0.022052266200383506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,7168,0.02691520055135091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,7168,0.02749119997024536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,6144,0.023093332846959434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,6144,0.02561066746711731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,512,0.005941333373387655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,5120,0.018926932414372762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,5120,0.023817600806554158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,5120,0.015833600362141927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,4096,0.015447466572125753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,4096,0.0219050665696462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,5120,256,0.005604266623655955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,3584,0.013843199610710144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,3584,0.021258666117986044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,65536,0.15340372721354167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,3072,0.013540266950925191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,51200,0.11859520276387532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,3072,0.011351466178894043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,2560,0.010633599758148194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,2560,0.019667200247446694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,12288,0.030983465909957885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,2048,0.00916266640027364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,10240,0.026726400852203368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,2048,0.018337066968282065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,1536,0.007837866743405659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,1536,0.01644373337427775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,7168,0.020474666357040407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,6144,0.018232532342274985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,1024,0.006253866851329804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,1024,0.01609599987665812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,3072,0.0206389327843984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,768,0.004419200122356415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,4096,0.013421866297721862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,768,0.015822933117548624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,768,0.006620799998442332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,512,0.003853866706291834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,512,0.015482667088508605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,256,0.0035349334279696143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,256,0.014967466394106546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,3584,0.012932266791661581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,128,0.003197866678237915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,128,0.014797866344451904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,128,0.005075199902057648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,64,0.00308693324526151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,64,0.015064533551534018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,4096,32,0.0034261333445707956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,4096,32,0.015102932850519816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,65536,0.18553706804911296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,2560,0.01056106686592102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,65536,0.12779946327209474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,51200,0.14684906005859374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,2048,0.009471999605496724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,51200,0.10373653570810955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,16384,0.049542399247487386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,1536,0.007820799946784973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,16384,0.04464106559753418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,12288,0.03963306744893392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,12288,0.03561493158340454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,10240,0.03456426858901977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,10240,0.03176640073458354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,1024,0.006726400057474773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,8192,0.02893120050430298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,8192,0.02773333390553792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,512,0.005776000022888183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,7168,0.02251946727434794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,4096,256,0.0054624001185099285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,7168,0.026115200916926068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,6144,0.020119466384251914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,6144,0.024678399165471397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,65536,0.14098347028096517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,5120,0.01687999963760376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,5120,0.022873600323994957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,5120,0.014866133530934652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,4096,0.01381119986375173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,51200,0.11015040079752605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,4096,0.02130026618639628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,4096,0.012578133742014566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,3584,0.012386133273442585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,3584,0.020871466398239134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,3072,0.011047466595967611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,16384,0.03778133392333984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,3072,0.020023467143376668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,2560,0.009831466277440389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,12288,0.029754666487375896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,2560,0.01926079988479614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,2048,0.008622933427492778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,2560,0.010046933094660442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,2048,0.018074667453765868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,1536,0.0073066666722297665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,10240,0.02534719904263814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,1536,0.01646613379319509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,1024,0.00518506666024526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,1024,0.01576959987481435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,8192,0.021226666371027627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,768,0.00439680020014445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,768,0.01588053305943807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,7168,0.01922666629155477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,512,0.0038133333126703895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,512,0.01553813318411509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,6144,0.0173418660958608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,256,0.003489066660404205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,256,0.015005866686503092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,3584,0.012227200468381246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,128,0.0031637333333492277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,3072,0.011057066917419433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,2048,0.008846933643023174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,64,0.0029898665845394133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,64,0.014890666802724203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3584,32,0.003151999910672506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,1536,0.007640533149242401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,32,0.014913066228230795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,1024,0.006528000036875407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,65536,0.16225066184997558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,65536,0.11392853260040284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,768,0.00639573335647583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,51200,0.12921600341796874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,51200,0.09608000119527181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,512,0.005853866537412008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,16384,0.04795519908269246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,12288,0.03498026529947917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,12288,0.03318613370259603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,256,0.005381333331267038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3584,128,0.014839466412862143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,10240,0.030487465858459472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,10240,0.02982826630274455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,10240,0.024889600276947022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,8192,0.025602134068806966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,8192,0.026130133867263795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,16384,0.04178986549377441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,7168,0.020323199033737183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,7168,0.02515520056088765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3584,128,0.005123200019200643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,6144,0.018119466304779053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,6144,0.023540266354878745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,6144,0.017063466707865398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,5120,0.017454934120178223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,5120,0.022191999355951945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,65536,0.14456639289855958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,4096,0.01532373329003652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,4096,0.021016534169514975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,4096,0.012693333625793456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,51200,0.11822506586710613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,3584,0.01147093375523885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,3584,0.0204586664835612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,16384,0.03682560125986735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,3072,0.010310399532318115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,3072,0.01970240076382955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,2560,0.009246933460235595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,12288,0.029425066709518433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,2560,0.018516266345977785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,2048,0.00798826664686203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,2048,0.017056000232696534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,8192,0.020915200312932335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,1536,0.006837333242098491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,1536,0.016365866859753928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,1024,0.004878933231035868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,1024,0.015884799758593242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,7168,0.018895999590555827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,768,0.004435199995835622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,768,0.015778133273124696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,5120,0.015222400426864624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,512,0.003772799919048945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,512,0.01564479966958364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,3584,0.011848533153533935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,256,0.0034688000877698263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,3072,0.010755200187365215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,256,0.01497066617012024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,128,0.0031818665564060213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,128,0.014697600404421488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,2560,0.010097066561381023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,64,0.003020799905061722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,64,0.014631467064221701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,2048,0.008547199765841167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,3072,32,0.0031583999594052637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,3072,32,0.014868266383806863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,65536,0.13921492894490559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,65536,0.10579733053843181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,1536,0.0075328002373377485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,1024,0.006737066805362702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,51200,0.08236800034840902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,16384,0.03760853211085002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,16384,0.03835626840591431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,768,0.006483200192451477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,12288,0.030375466744105024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,12288,0.031600000460942586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,51200,0.10270933310190837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,10240,0.026658133665720625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,10240,0.027832533915837603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,512,0.005660800139109293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,8192,0.022689066330591836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,8192,0.02498240073521932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,256,0.005301333467165629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,7168,0.018194133043289186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,3072,128,0.005075199902057648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,7168,0.023883734146753946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,6144,0.01590079963207245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,6144,0.0232149342695872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,65536,0.12863893508911134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,51200,0.10188373724619548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,5120,0.0135754664738973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,5120,0.02150933345158895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,4096,0.011451733112335206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,4096,0.020536533991495767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,16384,0.03464320103327433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,3584,0.010537599523862202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,3584,0.01995733380317688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,12288,0.027211733659108478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,3072,0.009607467055320739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,3072,0.019475199778874717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,10240,0.02332586646080017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,2560,0.008634666601816814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,2560,0.017372800906499227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,8192,0.019530665874481202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,2048,0.007529599964618683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,2048,0.016801067193349204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,2048,0.008161066472530365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,1536,0.0061482667922973635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,1536,0.016530133287111917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,7168,0.017887999614079796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,1024,0.004643199841181437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,1024,0.015602133671442666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,6144,0.016309332847595216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,768,0.004304000238577525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,768,0.016344533363978068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,5120,0.014363732933998109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,512,0.0037632000943024955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,4096,0.01206933359305064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,512,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,512,0.005643733342488607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,256,0.0034783999125162757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,3584,0.01130239963531494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,256,0.01479680041472117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,128,0.003160533308982849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,128,0.014664533734321594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,3072,0.010259200135866802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,64,0.0029824001093705496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,64,0.015024000406265258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2560,32,0.003045333425203959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2560,32,0.014900267124176025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,65536,0.10931519667307536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,2560,0.009113599856694538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,65536,0.08875306447347006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,65536,0.1210250695546468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,51200,0.08695360024770102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,51200,0.07353493372599283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,16384,0.03143360018730164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,1536,0.00724480003118515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,16384,0.03409599860509237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,12288,0.025719465812047322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,16384,0.03339200019836426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,1024,0.00655680000782013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,12288,0.03219199975331624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,10240,0.022443733612696328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,10240,0.029625600576400755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,8192,0.018380800882975258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,8192,0.027584000428517656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,768,0.006211199859778086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,7168,0.016264533003171287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,7168,0.025448532899220784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,7168,0.017692800362904867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,6144,0.015822933117548624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,6144,0.024094933271408082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,5120,0.013913599650065103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,256,0.005329066514968872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2560,128,0.005073066552480062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,5120,0.020890667041142782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,4096,0.012573867042859396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,4096,0.019745065768559774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,3584,0.011121066411336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,3584,0.019326933224995933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,51200,0.09980266888936361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,3072,0.00969599982102712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,3072,0.017917867501576742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,12288,0.025550933678944905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,10240,0.022530132532119752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,2560,0.007972266773382823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,2560,0.017485866943995156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,2048,0.007004799942175548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,8192,0.018778665860493978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,2048,0.01697173317273458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,1536,0.005353599786758423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,1536,0.01622293293476105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,6144,0.016127999623616537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,1024,0.004500266909599304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,1024,0.01609493295351664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,4096,0.011664000153541566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,5120,0.013564800222714743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,768,0.004108799993991852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,768,0.006155733267466227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,768,0.01590826710065206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,512,0.003734400123357773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,512,0.015683199961980185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,256,0.0033183999359607695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,256,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,3072,0.009654399752616883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,3584,0.011371733744939168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,128,0.003049599876006444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,128,0.014734933773676554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,64,0.00290133332212766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,64,0.01461013356844584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,2048,32,0.0029653333127498626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,2560,0.008820266524950663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,2048,32,0.014629333217938741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,2048,0.008215466638406117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,65536,0.08573333422342935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,51200,0.06866026719411214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,65536,0.07674986521402995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,51200,0.06392426490783691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,16384,0.02566080093383789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,16384,0.029986133178075153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,1024,0.006303999821345012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,12288,0.020243199666341145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,12288,0.027102933327356978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,512,0.005553066730499268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,10240,0.01808639963467916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,10240,0.025446399052937822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,10240,0.019563732544581096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,256,0.0052042668064435325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,8192,0.014983466267585755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,1536,0.00732479989528656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,8192,0.023220266898473105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,7168,0.013613866766293845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,7168,0.02290453314781189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,6144,0.012477866808573405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,6144,0.02174933354059855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,2048,128,0.004937600096066793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,65536,0.10540800094604492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,5120,0.01111466685930888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,5120,0.020917334159215293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,51200,0.08545066515604655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,4096,0.009610666831334432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,4096,0.019318399826685588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,16384,0.030112000306447347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,3584,0.008922666311264038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,3584,0.01825173298517863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,12288,0.02267306645711263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,3072,0.008193066716194153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,3072,0.017898666858673095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,8192,0.01733120083808899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,2560,0.007524266839027405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,2560,0.017526400089263917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,2560,0.008250666658083598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,2048,0.006634666522343953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,7168,0.01595946649710337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,2048,0.016908800601959227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,1536,0.005598933498064677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,1536,0.016360533237457276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,6144,0.01441493332386017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,1024,0.004699733356634776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,1024,0.016127999623616537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,768,0.004388266801834106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,768,0.015672533710797628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,5120,0.012311466534932454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,768,0.005743999779224396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,512,0.003925333420435587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,512,0.015406933426856995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,256,0.0033514666060606635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,256,0.015505066514015198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,4096,0.009987200299898785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,128,0.0030303999781608583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,128,0.014582399527231851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,3584,0.009608532985051472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,64,0.002994133283694585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,64,0.014834133783976236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1536,32,0.0031008000175158186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1536,32,0.01498240033785502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,65536,0.05706559816996256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,3072,0.008707200487454731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,65536,0.06486719846725464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,51200,0.04642773469289144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,65536,0.10233279863993328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,51200,0.054622932275136316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,51200,0.08027199904123941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,16384,0.01817493240038554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,16384,0.02547093431154887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,16384,0.028037333488464357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,12288,0.014328533411026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,2048,0.007559466858704884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,12288,0.023135999838511147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,10240,0.012657066186269125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,10240,0.022536534070968627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,10240,0.01885546644528707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,8192,0.011147733529408772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,1536,0.006857599814732869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,7168,0.012460800011952718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,7168,0.023796266317367552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,1024,0.005932799975077311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,6144,0.00967039962609609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,6144,0.022725333770116173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,6144,0.012779733538627625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,5120,0.009524266918500264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,5120,0.020268799861272176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,8192,0.021575466791788737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,512,0.005381333331267038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,4096,0.011756799618403117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,4096,0.018333866198857626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,4096,0.009717333316802978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,3584,0.010824533303578694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,7168,0.014689067006111145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,3584,0.018595200777053834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,3072,0.009659733374913533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,3072,0.01762239933013916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,3072,0.008454400300979614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,256,0.005047466854254405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,2560,0.007212799787521362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,2048,0.0061365331212679545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,2560,0.008203733464082081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,2560,0.017211733261744182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,1536,0.0052906667192777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,2048,0.016615466276804606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1536,128,0.004824533561865489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,1536,0.006906666855017345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,1024,0.004456533491611481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,1536,0.016310399770736693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,768,0.004051200052102407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,1024,0.005931733548641205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,1024,0.016035200158754984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,512,0.0037216000258922578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,768,0.015289599696795145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,512,0.005266133447488149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,512,0.015362133582433065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,256,0.003188266605138779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,256,0.014740266402562461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,256,0.004890666902065277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,128,0.0030901332696278887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,128,0.014617600043614707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,128,0.004780800143877665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,64,0.0028661333024501802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,64,0.014873600006103516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,1024,32,0.0028917332490285236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,1024,32,0.014792533715566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,65536,0.046462933222452804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,65536,0.059621334075927734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,12288,0.02183893322944641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,51200,0.03816426595052083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,65536,0.09753812948862711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,8192,0.01618666648864746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,16384,0.01441493332386017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,51200,0.04924053351084391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,16384,0.027088000377019243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,16384,0.024471465746561685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,12288,0.011710932850837708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,12288,0.021128533283869426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,10240,0.011509333054224651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,10240,0.02448319991429647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,10240,0.018224000930786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,8192,0.010148266951243084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,8192,0.023044266303380332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,8192,0.01520639955997467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,5120,0.011266133189201355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,7168,0.02102400064468384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,7168,0.013852799932161966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,6144,0.008900266885757447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,6144,0.019801600774129232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,6144,0.012457600235939026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,5120,0.01046399970849355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,5120,0.019789866606394448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,5120,0.011170132954915365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,4096,0.008754133184750875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,4096,0.019051732619603474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,4096,0.009638399879137675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,3584,0.00806826651096344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,3584,0.018289067347844443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,12288,0.023201066255569457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,3584,0.0093642661968867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,3072,0.006939733525117238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,3584,0.009461333354314167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,2560,0.0074421331286430355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,3072,0.008434133728345235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,3072,0.017654399077097573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,2048,0.00574186642964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,2560,0.00812906672557195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,2560,0.01739733417828878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,2048,0.007159466544787089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,1536,0.004987733562787374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,2048,0.016673066218694053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,1024,0.004344533383846283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,1536,0.006751999755700429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,2048,0.007336533566315968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,1536,0.01618133286635081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,768,0.0039989332358042395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,1024,0.005955199897289276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,1024,0.015583999951680503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,768,0.015495466192563376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,512,0.0035285333792368567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,7168,0.009660800298055012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,1024,768,0.005532800157864889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,512,0.005273599922657013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,256,0.0031530665854612983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,512,0.015200000007947287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,128,0.002930133293072383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,256,0.014802133043607077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,256,0.004941866795221964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,64,0.0028010666370391845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,128,0.004814933240413666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,768,32,0.0028010666370391845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,128,0.014757333199183145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,64,0.014739200472831726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,768,32,0.014705066879590353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,65536,0.03465813398361206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,51200,0.028180267413457232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,65536,0.05172266562779745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,16384,0.012845866878827415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,65536,0.0925930658976237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,51200,0.07178880373636881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,12288,0.01016426682472229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,16384,0.02651946743329366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,16384,0.023389865954717003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,10240,0.009954133629798889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,12288,0.022507733106613158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,12288,0.02074133356412252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,10240,0.02118826707204183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,8192,0.008949333429336548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,8192,0.019630932807922365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,7168,0.00811839997768402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,7168,0.019372800985972084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,6144,0.00902400016784668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,7168,0.013697066903114319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,6144,0.01899626652399699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,5120,0.007783466577529907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,6144,0.012261333068211873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,4096,0.007091199855009715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,5120,0.011043199896812439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,5120,0.019383466243743895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,3584,0.006773333251476288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,4096,0.009621333082516987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,4096,0.01843199928601583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,3584,0.017941333850224814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,3072,0.0064181332786877945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,3072,0.01730133295059204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,3072,0.008413867155710856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,51200,0.07540799776713053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,2560,0.00626986672480901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,2560,0.007922133306662242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,2560,0.01688106656074524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,2048,0.00552106648683548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,2048,0.007216000060240428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,2048,0.016427733500798545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,1536,0.00483840008576711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,768,768,0.00565119981765747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,1536,0.016170666615168253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,1024,0.004250666499137879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,1024,0.015550933281580605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,1024,0.005743999779224396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,768,0.0037994667887687682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,768,0.01581546664237976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,768,0.005554133156935374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,512,0.0034517332911491393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,512,0.005160533388455709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,512,0.015037866433461508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,51200,0.04290879964828491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,256,0.0031968000034491217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,256,0.004839466512203216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,256,0.01562346617380778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,10240,0.01731200019518534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,128,0.004755199948946635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,128,0.014745600024859109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,64,0.0028160000840822858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,32,0.0028597332537174227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,64,0.014482133587201438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,65536,0.02250880002975464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,512,32,0.014517333110173544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,51200,0.01737066706021627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,51200,0.03532906770706177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,3584,0.009286399682362874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,16384,0.00897279977798462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,16384,0.02223680019378662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,16384,0.02490560015042623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,12288,0.008337066570917765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,12288,0.020819199085235596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,12288,0.019735467433929444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,10240,0.007288533449172974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,10240,0.019573332866032918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,10240,0.016956800222396852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,8192,0.00801813354094823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,8192,0.019012266397476198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,8192,0.014446933070818582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,7168,0.007501866420110066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,512,128,0.002977066735426585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,7168,0.01913493275642395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,6144,0.006988800068696339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,6144,0.01876693367958069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,6144,0.012198399504025776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,65536,0.04461866617202759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,5120,0.006555733581384023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,5120,0.010931199789047242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,5120,0.018887466192245482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,8192,0.01476800044377645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,4096,0.01803413430849711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,3584,0.0066997334361076355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,3584,0.0176256000995636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,3584,0.009332266449928284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,3072,0.00625600020090739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,3072,0.01734506686528524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,3072,0.008337066570917765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,2560,0.006116266548633576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,2560,0.017102932929992674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,2048,0.005401599903901418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,2048,0.01646080017089844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,2048,0.007133866846561432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,1536,0.004791466891765595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,512,1536,0.006663466493288676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,1536,0.015940266847610473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,1024,0.004116266717513402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,1536,0.006774400174617767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,1024,0.005714133381843567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,1024,0.015566933155059814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,768,0.0036415999134381616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,768,0.015544533729553223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,512,0.003389866650104523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,512,0.015201066931088766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,4096,0.006131199995676676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,256,0.0030048000315825146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,512,0.005182933310667674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,4096,0.00974826713403066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,256,0.00486826648314794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,128,0.0028959999481836954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,256,0.01479680041472117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,64,0.0026602665583292644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,128,0.004644266764322917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,128,0.014514133334159851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,256,32,0.0027488000690937043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,65536,0.08897173404693604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,32,0.014569600423177084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,65536,0.013648000359535218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,51200,0.011896533767382304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,65536,0.09017066955566407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,51200,0.031897600491841635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,51200,0.07173759937286377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,16384,0.008413867155710856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,16384,0.020873600244522096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,16384,0.025254400571187337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,12288,0.007626666625340779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,12288,0.0199455996354421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,12288,0.019911466042200725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,10240,0.0070250665148099255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,768,0.00551146666208903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,10240,0.020850133895874024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,10240,0.017193599541982015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,8192,0.006487466891606649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,8192,0.014632532993952433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,7168,0.006235733131567637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,7168,0.019406932592391967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,7168,0.0135861337184906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,256,64,0.014532267053922018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,6144,0.006035199761390686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,6144,0.012288000186284382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,6144,0.01863893270492554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,65536,0.03853653271993001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,5120,0.00644053320089976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,5120,0.018798933426539103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,4096,0.006131199995676676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,4096,0.009543466567993163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,4096,0.018339200814565023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,3584,0.0065184002121289565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,3584,0.01763733426729838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,3584,0.009297066926956176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,3072,0.006151466568311056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,3072,0.01735573410987854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,3072,0.008220799763997396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,2560,0.006109866499900818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,2560,0.016746666034062704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,2560,0.007895466685295106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,2048,0.00537066658337911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,2048,0.016746666034062704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,2048,0.007028266787528992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,1536,0.004730666677157084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,2560,0.007960533102353413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,1536,0.01608746647834778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,1024,0.004009599983692169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,1024,0.01555519998073578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,768,0.0036490666369597114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,768,0.01539520025253296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,768,0.005397333204746247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,512,0.0034175999462604523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,512,0.014958932995796204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,8192,0.019458132982254028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,51200,0.0696938673655192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,512,0.005131733417510986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,256,0.0030250666042168934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,128,0.002810666710138321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,256,0.014632532993952433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,128,0.01469013293584188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,64,0.0026357332865397137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,64,0.01451520025730133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,128,32,0.002657066782315572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,128,32,0.014318933089574179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,5120,0.011011200149854024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,65536,0.010446932911872864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,51200,0.009478400150934856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,65536,0.03526186545689901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,16384,0.00673280010620753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,51200,0.029637332757314044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,12288,0.006186666587988535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,16384,0.020524799823760986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,10240,0.006371200084686279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,12288,0.02021119991938273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,8192,0.006303999821345012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,10240,0.020551466941833497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,7168,0.006078933179378509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,8192,0.018994132677714028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,6144,0.005958400170008342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,7168,0.019285333156585694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,1024,0.0058794667323430385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,4096,0.005950933198134104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,5120,0.019371734062830607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,3584,0.006458666423956554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,3584,0.017770665884017944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,3072,0.006133333345254262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,3072,0.01769599914550781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,2560,0.006035199761390686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,2560,0.017115734020868936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,128,0.0047082667549451195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,2048,0.005383466680844625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,1536,0.004698666433493296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,2048,0.016149333119392394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,1024,0.004018133382002512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,1536,0.01613653302192688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,768,0.0036373332142829893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,768,0.01553813318411509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,512,0.003403733422358831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,512,0.014856533209482829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,256,0.0030389333764712016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,256,0.01460906664530436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,128,0.0029056000212828318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,128,0.014471466342608133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,256,7168,0.013525333007176718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,64,0.002678400029738744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,64,0.014620799819628397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,4096,0.017986132701237997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,32,0.0026719999810059865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,32,0.014446933070818582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,65536,0.013393066326777139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,51200,0.011224533120791119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,65536,0.03528106609980265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,51200,0.030317866802215578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,16384,0.006760533154010773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,12288,0.006197333335876465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,16384,0.020702934265136717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,12288,0.01993173360824585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,10240,0.019705599546432494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,8192,0.006233599781990051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,1024,0.015125333269437154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,8192,0.019245866934458414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,7168,0.006110933423042297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,256,0.004835199813048045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,6144,0.005907199780146281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,7168,0.019314134120941163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,5120,0.006361599763234456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,6144,0.018886399269104005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,4096,0.005949866771697998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,5120,0.018905599912007652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,3584,0.006411733229955037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,4096,0.0187285323937734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,3584,0.017516799767812095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,2560,0.006044800082842508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,3072,0.017691733439763387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,2048,0.005299200117588043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,2560,0.016832000017166136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,1536,0.004691199958324432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,2048,0.01667520006497701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,1024,0.003960533440113068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,1536,0.01593066652615865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,768,0.0036874666810035707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,1024,0.015545599659283958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,512,0.0032960000137488045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,64,5120,0.006424533327420552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,768,0.01537493367989858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,256,0.0030847998956839246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,512,0.015000533064206442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,128,0.0028629332780838014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,256,0.014625066518783569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,64,0.0026389333109060925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,128,0.01434346636136373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,32,0.002682666728893916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,64,0.014476799964904785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,32,32,0.014393599828084311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,10240,0.00636053333679835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,112,32,3072,0.00609493354956309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,112,64,6144,0.018992000818252565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,16384,0.7614816029866536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,112,128,1536,0.006637866795063019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,16384,0.40442454020182295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,16384,0.3729962666829427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,12288,0.5793162663777669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,12288,0.3106133460998535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,10240,0.2612213293711344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,51200,1.213473129272461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,51200,1.142401123046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,10240,0.4915466626485189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,8192,0.2146677335103353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,10240,0.23748265902201332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,8192,0.3874261220296224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,7168,0.1886613368988037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,8192,0.22179946899414063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,7168,0.1732565402984619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,7168,0.39559787114461265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,6144,0.2943829218546549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,6144,0.17657492955525717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,51200,2.3701876322428386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,6144,0.14887466430664062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,5120,0.14114987055460612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,5120,0.143231995900472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,4096,0.1996448040008545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,3584,0.17562665939331054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,3584,0.10524799823760986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,12288,0.2825279871622721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,4096,0.1160746653874715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,3584,0.09324800173441569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,3072,0.15189226468404132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,3072,0.09377386569976806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,3072,0.08214186827341716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,2560,0.12758933703104655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,2560,0.07039893468221028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,2048,0.10387732982635497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,2560,0.08988906542460123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,2048,0.06906346480051676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,2048,0.0587007999420166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,1536,0.08066986401875814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,1024,0.04450026750564575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,1024,0.06341226498285929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,1536,0.046804265181223555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,768,0.04405866861343384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,1024,0.03504213492075602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,512,0.03234666585922241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,256,0.018497065703074137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,768,0.03799893458684285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,512,0.03065599997838338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,512,0.023835732539494833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,768,0.03317226568857829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,128,0.010782933235168457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,256,0.017672532796859743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,256,0.02342826724052429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,65536,128,0.01599679986635844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,128,0.021499733130137123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,64,0.01971199909845988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,32,0.01946559945742289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,5120,0.24573225975036622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,4096,0.11782399813334148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,65536,1536,0.05631253321965536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,64,0.008345599969228108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,65536,32,0.007551999886830647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,16384,0.6019168217976888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,65536,1.2335700988769531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,51200,0.9671178817749023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,51200,1.8701216379801433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,16384,0.321885871887207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,51200,0.9391232172648112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,65536,2.447857157389323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,16384,0.3204085350036621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,12288,0.46629654566446943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,12288,0.22853439648946128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,12288,0.28568747838338215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,8192,0.30491307576497395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,10240,0.21145599683125815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,10240,0.38317972819010415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,8192,0.17218133608500164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,7168,0.15235840479532878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,7168,0.1428426742553711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,7168,0.2743040084838867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,6144,0.23395946820576988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,6144,0.1349013328552246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,6144,0.12115626335144043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,5120,0.19262933731079102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,5120,0.11514453093210857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,4096,0.1563488006591797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,4096,0.09706026713053385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,3584,0.08723413149515788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,3584,0.1387466589609782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,3072,0.07785493532816569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,3072,0.12073813279469807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,2560,0.10216746330261231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,2560,0.0687018632888794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,65536,1.2023541768391928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,2048,0.09527466297149659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,2048,0.06384106477101645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,2048,0.051804800828297935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,1536,0.06529920101165772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,1536,0.049505066871643064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,10240,0.22114666302998862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,8192,0.16123305956522624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,1024,0.05046720107396444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,1024,0.041516800721486405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,1024,0.03177280028661092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,768,0.036200535297393796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,768,0.03532053232192993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,512,0.029679999748865767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,512,0.029178667068481445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,5120,0.10784107049306232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,4096,0.08406399885813395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,256,0.01686933239301046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,3584,0.07440746625264485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,256,0.023656533161799113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,128,0.01088746686776479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,128,0.020524799823760986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,64,0.007933866480986278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,128,0.012121599912643433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,51200,32,0.007277866701285045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,64,0.019370667139689126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,3072,0.06535786787668864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,51200,32,0.019400533040364584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,2560,0.0569045344988505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,65536,0.37734079360961914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,65536,0.7724490483601888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,51200,0.5694485346476237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,51200,0.3158090591430664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,16384,0.2141258716583252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,51200,0.3040362675984701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,16384,0.10133013725280762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,16384,0.11476159890492757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,1536,0.03791999816894531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,12288,0.0912448008855184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,12288,0.07806293169657388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,12288,0.16425600051879882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,10240,0.12059413592020671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,10240,0.07810133298238119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,10240,0.06648639837900797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,8192,0.1025322675704956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,8192,0.0670464038848877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,65536,0.39906028111775715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,8192,0.05505173206329346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,7168,0.08552746772766114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,768,0.0243776003519694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,7168,0.059971201419830325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,6144,0.0540832002957662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,5120,0.04781333208084106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,6144,0.04372906684875488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,5120,0.06294826666514078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,5120,0.03777066469192505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,4096,0.052153599262237546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,4096,0.041783467928568525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,4096,0.032008532683054605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,3584,0.0458901325861613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,3072,0.04031360149383545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,3584,0.029028266668319702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,3584,0.03850133419036865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,512,0.0189301331837972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,2560,0.03418879906336467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,3072,0.03553920189539592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,51200,256,0.014249599973360696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,2048,0.02850346763928731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,2560,0.03254933357238769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,2048,0.01955946683883667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,1536,0.022679466009140014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,2048,0.028653866052627562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,1536,0.02508373260498047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,1536,0.016104533274968465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,1024,0.015915733575820924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,1024,0.01286720037460327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,768,0.012414933244387309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,1024,0.0221450666586558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,512,0.00942186713218689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,768,0.011566932996114094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,768,0.020232532421747842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,256,0.0065749332308769224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,512,0.009801600376764934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,512,0.01871466636657715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,128,0.004324266811211904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,256,0.007362133264541626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,256,0.016123732924461363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,64,0.004033066580692927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,128,0.015901866555213928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,32,0.004337066908677419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,64,0.016114133596420287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,16384,32,0.016280532876650492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,16384,6144,0.08683733145395914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,65536,0.3265834808349609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,65536,0.5828906377156575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,65536,0.3029088020324707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,128,0.0069354668259620665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,51200,0.2658549308776855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,51200,0.45244480768839523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,51200,0.23867413202921547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,16384,0.15762346585591633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,16384,0.09332799911499023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,12288,0.12279573281606038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,12288,0.0743658701578776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,12288,0.06201279958089193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,7168,0.049951998392740886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,10240,0.09134720166524252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,10240,0.06554986635843912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,10240,0.05324800014495849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,8192,0.073526398340861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,8192,0.0558240016301473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,7168,0.06636693477630615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,8192,0.044420266151428224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,7168,0.04973866542180379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,6144,0.058823466300964355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,6144,0.04573226769765218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,6144,0.03460373481114705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,5120,0.04969919919967651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,5120,0.041033601760864256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,4096,0.04129600127538045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,4096,0.036644268035888675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,3072,0.02622399926185608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,16384,2560,0.023086933294932048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,4096,0.025568000475565594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,3584,0.03516693512598674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,3584,0.03399253288904826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,3072,0.031064534187316896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,3584,0.023005867004394533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,3072,0.031531733274459836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,3072,0.020871466398239134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,2560,0.028486400842666626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,2048,0.02214399973551432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,2560,0.026590933402379353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,2048,0.025684267282485962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,1536,0.013133866588274637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,1024,0.012201600273450216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,1536,0.02330346703529358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,1024,0.010838400324185688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,1024,0.020694400866826376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,768,0.010037333766619364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,768,0.01953386664390564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,512,0.007997866471608479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,512,0.01730133295059204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,512,0.008014933268229166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,256,0.004682666560014089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,256,0.01590720017751058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,256,0.006492800017197926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,128,0.003969066590070724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,128,0.015362133582433065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,128,0.005952000121275584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,64,0.0036309334139029183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,64,0.015804800391197204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,32,0.0038901334007581077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,2560,0.01827413241068522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,12288,1536,0.017511467138926186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,12288,32,0.01577173372109731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,16384,0.09172373612721761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,65536,0.28627198537190757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,65536,0.49563414255778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,65536,0.26709651947021484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,51200,0.23007465998331705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,7168,0.039545599619547525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,51200,0.3913642565409342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,16384,0.1381173292795817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,5120,0.03027413288752238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,16384,0.08009386857350667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,16384,0.07927146752675375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,12288,0.09288000265757243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,12288,0.06411946614583333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,10240,0.07802133560180664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,10240,0.047414398193359374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,10240,0.056066131591796874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,8192,0.06262400150299072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,2048,0.015430399775505066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,8192,0.04889599879582723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,8192,0.03921600182851155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,7168,0.055870934327443444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,7168,0.04461866617202759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,7168,0.03540053367614746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,6144,0.04932159980138143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,6144,0.031164799133936567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,6144,0.040617601076761885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,5120,0.04169066747029622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,12288,768,0.009709866841634114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,5120,0.0368778665860494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,4096,0.03431040048599243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,5120,0.02748799920082092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,4096,0.032407466570536295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,3584,0.03062506715456645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,4096,0.022308266162872313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,3584,0.020125865936279297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,3584,0.030289065837860108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,3072,0.026943999528884887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,3072,0.02791253328323364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,3072,0.017875200510025023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,2560,0.02373973329861959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,2560,0.025491199890772503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,2048,0.01360106666882833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,2048,0.023483733336130776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,1536,0.015178666512171427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,1024,0.011078400413195293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,1536,0.011685333649317424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,1536,0.021517866849899293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,768,0.009318400422732036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,1024,0.010228266318639118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,1024,0.01984106699625651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,512,0.007609599828720092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,768,0.008664533495903015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,512,0.01643519997596741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,256,0.004572799801826477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,256,0.015910399953524272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,256,0.006098133325576782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,128,0.003933866570393244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,128,0.015595733126004537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,64,0.003607466568549474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,12288,0.055111467838287354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,32,0.0036821333070596062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,64,0.01572266618410746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,32,0.015556266903877259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,65536,0.22909013430277506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,10240,2048,0.019342933098475137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,65536,0.4082101186116536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,10240,768,0.018116267522176106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,51200,0.18399786949157715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,51200,0.29915733337402345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,16384,0.10998079776763917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,16384,0.06715733210245768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,51200,0.21582719484965004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,12288,0.07384533087412516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,12288,0.059443199634552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,10240,0.061909333864847815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,10240,0.04806400140126546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,10240,0.04122453530629476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,8192,0.05038079818089804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,8192,0.041560534636179605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,2560,0.01600213348865509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,7168,0.04465493361155192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,7168,0.038533333937327066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,512,0.007098666826883952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,6144,0.03957013289133708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,6144,0.035257601737976076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,10240,128,0.0054506664474805195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,5120,0.03344213167826335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,5120,0.03168639938036601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,5120,0.023430399099985757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,4096,0.02765333255132039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,4096,0.028266666332880656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,4096,0.019719467560450236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,3584,0.02477759917577108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,3584,0.026308266321818034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,65536,0.23458879788716636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,3072,0.024710400899251302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,3072,0.024820266167322795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,51200,0.19093972841898602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,16384,0.06347413460413615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,2560,0.018773333231608073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,12288,0.04784853458404541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,2560,0.023179733753204347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,2048,0.015476266543070475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,2560,0.014662399888038635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,2048,0.01270080010096232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,1536,0.012066133817036946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,1536,0.019961599508921304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,1536,0.011025066177050274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,1024,0.009071999788284301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,1024,0.01867626706759135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,8192,0.03407786687215169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,768,0.007646933197975159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,768,0.016587733229001363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,7168,0.03071146607398987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,512,0.006227200229962667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,512,0.01639359990755717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,6144,0.027118933200836182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,256,0.004223999877770742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,256,0.015412267049153647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,2048,0.021465599536895752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,256,0.00624533345301946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,128,0.0037429332733154297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,128,0.015252266327540079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,64,0.0034154665966828666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,64,0.01555519998073578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,8192,32,0.0038783999780813852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,8192,32,0.01566933294137319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,3584,0.018343466520309448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,65536,0.21371520360310875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,65536,0.3546250661214193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,51200,0.2781536102294922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,3072,0.017097600301106772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,51200,0.17306772867838544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,16384,0.09524799982706705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,16384,0.060976000626881924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,1024,0.009596799810727436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,12288,0.06511893272399902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,12288,0.051062401135762533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,10240,0.05492800076802572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,768,0.008590933680534363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,10240,0.0447872002919515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,8192,0.04453440109888713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,512,0.007313066720962524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,8192,128,0.0058218667904535925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,7168,0.03941973447799683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,7168,0.03617493311564128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,8192,0.039272534847259524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,6144,0.03511786858240763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,6144,0.033140265941619874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,6144,0.02704533338546753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,65536,0.2319104035695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,5120,0.029850665728251142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,5120,0.02296746571858724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,4096,0.02477653423945109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,4096,0.026425600051879883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,51200,0.18849172592163085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,3584,0.02454293370246887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,3584,0.02505600055058797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,16384,0.06381440162658691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,3072,0.019492266575495402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,3072,0.02367466688156128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,12288,0.04632746775945028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,2560,0.016645333170890807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,2560,0.022258132696151733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,10240,0.04091200033823649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,2048,0.013557333747545877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,2048,0.020857600371042888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,8192,0.03364799817403157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,1536,0.010915199915568035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,5120,0.029917865991592407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,1536,0.010476799805959065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,1536,0.019462400674819948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,1024,0.008542933066685994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,1024,0.016745599110921223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,1024,0.008820266524950663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,768,0.007302399973074596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,768,0.01568000018596649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,512,0.004916266600290934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,512,0.01562879979610443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,7168,0.030776532491048177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,256,0.00401706670721372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,256,0.015438933173815408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,4096,0.018999467293421425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,128,0.003533866753180822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,128,0.015343999862670899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,3584,0.017002665996551515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,64,0.0035285333792368567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,3072,0.01567039986451467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,7168,32,0.003589333345492681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,64,0.015201066931088766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,2560,0.014105600118637086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,7168,32,0.015528532862663268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,2048,0.011897599697113037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,65536,0.3004085222880045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,65536,0.18601706822713215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,768,0.007606400052706401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,51200,0.2362229347229004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,16384,0.08462080160776773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,51200,0.15123945871988934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,512,0.006769066552321117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,16384,0.05525546471277872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,12288,0.06312426726023355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,16384,0.059772801399230954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,12288,0.0463648001352946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,10240,0.04949866533279419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,10240,0.040992001692454025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,8192,0.038412801424662274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,256,0.005734399954477946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,8192,0.03554346561431885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,7168,0.03448853492736816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,8192,0.031863466898600264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,7168,0.03331306576728821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,6144,0.030859732627868654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,6144,0.03041173418362935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,6144,0.025554132461547852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,5120,0.02611946662267049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,5120,0.027748266855875652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,5120,0.022056533892949422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,4096,0.021547732750574748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,4096,0.02474773327509562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,51200,0.174509859085083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,3584,0.022060799598693847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,3584,0.02418773372968038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,3584,0.016311466693878174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,3072,0.017422932386398315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,7168,128,0.005314133564631144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,3072,0.022784000635147093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,2560,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,2560,0.021581866343816123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,2560,0.013434666395187377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,2048,0.012657066186269125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,2048,0.02095359961191813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,65536,0.21843627293904624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,1536,0.011240532994270325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,1536,0.01917546590169271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,1024,0.008657067020734151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,1024,0.01665279964605967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,1024,0.00787306676308314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,12288,0.044197332859039304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,768,0.006903466582298279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,768,0.016051200032234193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,512,0.004932266473770141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,512,0.015561599532763162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,10240,0.039400533835093184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,512,0.006366933385531108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,256,0.003819733361403147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,256,0.015284267067909241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,256,0.005409066875775656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,128,0.003387733300526937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,7168,0.029292800029118854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,128,0.014912000298500061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,128,0.005182933310667674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,64,0.003271466741959254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,6144,32,0.003382399926582972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,64,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,6144,32,0.015346133708953857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,65536,0.24402880668640137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,65536,0.15769173304239908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,4096,0.017914666732152303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,51200,0.19250879287719727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,51200,0.12801706790924072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,16384,0.07047573725382486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,16384,0.05693973302841186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,12288,0.055752531687418616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,12288,0.0424074649810791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,3072,0.014802133043607077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,10240,0.0484063982963562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,10240,0.03757226864496867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,2048,0.01137600044409434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,8192,0.04095146656036377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,8192,0.03309226632118225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,1536,0.009954133629798889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,7168,0.0300437331199646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,7168,0.030631466706593828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,6144,768,0.0077909335494041445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,6144,0.026604799429575603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,6144,0.02805333336194356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,65536,0.2081610679626465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,5120,0.024817067384719848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,5120,0.025602134068806966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,51200,0.16773120562235516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,16384,0.056486399968465176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,4096,0.0186954657236735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,4096,0.023899734020233154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,3584,0.016935465733210246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,12288,0.042466131846110027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,3584,0.0228000005086263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,3072,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,3072,0.021553067366282146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,10240,0.03705600102742513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,2560,0.01258026659488678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,2560,0.020502400398254395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,2560,0.01286079982916514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,2048,0.010807466506958009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,2048,0.019363200664520262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,2048,0.010981333255767823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,1536,0.008912000060081481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,8192,0.03083733320236206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,1536,0.018154666821161906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,1536,0.009730133414268493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,1024,0.007075199981530507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,1024,0.01599360009034475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,7168,0.028062933683395387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,768,0.00550186683734258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,768,0.01616853376229604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,6144,0.023947733640670776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,512,0.004240000247955322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,512,0.015710933009783427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,5120,0.020836265881856282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,256,0.0037930667400360107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,256,0.015364266435305276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,4096,0.01714026729265849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,128,0.0034346667428811393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,128,0.014989866813023885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,128,0.005328000088532766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,64,0.003239466746648153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,3584,0.01563093364238739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,64,0.015123200416564942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,3072,0.014292266964912415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,5120,32,0.0032640000184377036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,5120,32,0.015185067057609558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,65536,0.1990645408630371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,65536,0.13658347129821777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,51200,0.15819840431213378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,51200,0.11149013042449951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,16384,0.05530986785888672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,51200,0.1455669403076172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,16384,0.044990932941436766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,12288,0.04407999912897746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,1024,0.008368000388145447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,12288,0.03772159814834595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,10240,0.03784106572469075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,10240,0.03335786660512288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,10240,0.033192533254623416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,8192,0.027693865696589152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,8192,0.02934719920158386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,8192,0.027538132667541505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,7168,0.024579199155171712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,7168,0.02704426646232605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,7168,0.024638932943344117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,6144,0.02219840089480082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,6144,0.0253983994325002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,6144,0.022026666005452476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,5120,0.018702934185663857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,5120,0.023976532618204753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,768,0.007277866701285045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,4096,0.0155349334081014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,4096,0.02221440076828003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,3584,0.01356160044670105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,512,0.00629013329744339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,3584,0.02133973240852356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,3584,0.014644267161687216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,3072,0.012098133563995361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,3072,0.02018346587816874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,2560,0.010594133536020916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,2560,0.019627734025319418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,5120,256,0.0054666668176651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,2048,0.009284266829490661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,2048,0.018770132462183634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,65536,0.19445652961730958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,1536,0.007890133559703827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,1536,0.01639253298441569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,16384,0.04973973433176677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,1024,0.006188799937566122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,1024,0.016385066509246825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,5120,0.019029333194096883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,768,0.004570666452248891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,768,0.015711999932924905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,4096,0.015925332903862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,512,0.004020266731580098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,512,0.015808000167210897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,3072,0.01349546710650126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,256,0.003509333233038584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,256,0.015238400300343832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,2560,0.011870933572451274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,128,0.003188266605138779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,128,0.01476693352063497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,12288,0.03733439842859904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,128,0.00513919989267985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,64,0.003020799905061722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,64,0.015236266454060874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,4096,32,0.003409066547950109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,4096,32,0.015326933066050211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,65536,0.18264533678690592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,65536,0.12906666596730548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,2048,0.01055680016676585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,65536,0.17003626823425294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,51200,0.14496533075968426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,51200,0.10618133544921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,16384,0.04901546637217204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,16384,0.04118933280309041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,16384,0.04771946668624878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,12288,0.03954026699066162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,12288,0.03527573347091675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,10240,0.0339466651280721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,10240,0.03154559930165608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,1536,0.00881813367207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,1024,0.007828266421953837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,8192,0.02476159930229187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,8192,0.02737706700960795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,768,0.006984533369541168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,7168,0.02223466634750366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,512,0.00626453310251236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,7168,0.025755733251571655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,7168,0.023998934030532836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,6144,0.019853866100311278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,6144,0.024247467517852783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,5120,0.017003732919692992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,5120,0.023110399643580117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,4096,256,0.005514666438102722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,4096,0.013708800077438354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,4096,0.02173973321914673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,4096,0.015244799852371215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,3584,0.012423466642697651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,3584,0.020787199338277183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,51200,0.1359861373901367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,3072,0.011181867122650147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,3072,0.02028800050417582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,12288,0.03507519960403442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,2560,0.010107733805974324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,2560,0.01957119901974996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,10240,0.031691733996073404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,2048,0.008824533224105835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,2048,0.01707093318303426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,1536,0.007665066421031952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,1536,0.016503467162450155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,8192,0.026969599723815917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,1024,0.005845333139101664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,1024,0.016355199615160625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,6144,0.021092265844345093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,768,0.0048096001148223875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,768,0.01591679950555166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,5120,0.018114133675893148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,512,0.004082133372624716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,512,0.015852800011634825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,3584,0.014084266622861228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,256,0.003659733384847641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,256,0.015618133544921874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,256,0.005426133175690969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,128,0.0033845332761605583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,128,0.014995200435320535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,128,0.0052255998055140175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,64,0.003151999910672506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,64,0.015079466501871744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3584,32,0.0032437334458033243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,3072,0.013010133306185404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,2560,0.011628799637158712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,65536,0.1581578731536865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,2048,0.009829333424568177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,65536,0.1152575969696045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,65536,0.19062827428181967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,51200,0.12622400124867755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,1536,0.008847999572753906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,51200,0.09273706277211508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,16384,0.04692266782124837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,16384,0.03909653425216675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,16384,0.05073173443476359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,12288,0.03466666539510091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,12288,0.03334186474482219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,10240,0.030133332808812457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,10240,0.029563732941945392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,8192,0.02212693293889364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,8192,0.02608746687571208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3584,32,0.015236266454060874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,1024,0.00765119989713033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,7168,0.020124799013137816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,7168,0.024701867500940958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,6144,0.018017067511876424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,6144,0.02381333311398824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,6144,0.022281599044799805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,768,0.006763733426729838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,5120,0.015262933572133383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,5120,0.02209920088450114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,4096,0.012337066729863485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3584,512,0.006084266801675161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,4096,0.020897066593170165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,3584,0.011457066734631855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,3584,0.01985493302345276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,3584,0.014460800091425577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,3072,0.01035093367099762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,3072,0.01944533387819926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,3072,0.013059199849764506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,2560,0.009257599711418152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,2560,0.018489599227905273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,51200,0.15290133158365887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,2048,0.0079925333460172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,12288,0.038474667072296145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,2048,0.017762132485707603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,2048,0.010054399569829304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,1536,0.006837333242098491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,10240,0.034329601128896076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,1536,0.01768853267033895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,1024,0.004915200173854828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,1024,0.01606613298257192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,768,0.004569600025812784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,8192,0.028039467334747315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,768,0.0160480002562205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,7168,0.0258240004380544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,512,0.003930666546026865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,512,0.015383467078208923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,512,0.006175999840100607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,256,0.003487999985615412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,256,0.01537493367989858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,5120,0.01977919936180115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,128,0.0033301333586374915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,4096,0.015864533185958863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,128,0.014918399850527444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,64,0.003127466638882955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,64,0.015129599968592325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,3072,32,0.0032458665470282235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,3072,32,0.015108266472816467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,65536,0.13702400525410968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,2560,0.012038399775822956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,65536,0.10116373697916667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,51200,0.11037546793619793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,51200,0.08017919858296713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,51200,0.13927359580993653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,16384,0.03772693475087484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,16384,0.037189332644144694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,1536,0.008664533495903015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,12288,0.030152533451716108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,1024,0.007482666770617168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,12288,0.0307861328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,12288,0.036236798763275145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,768,0.006805333495140076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,10240,0.02792746623357137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,8192,0.021972266832987468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,8192,0.0248799999554952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,7168,0.01767680048942566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,8192,0.02613226572672526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,7168,0.024488532543182374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,7168,0.02405440012613932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,6144,0.015655466914176942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,6144,0.022712532679239908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,5120,0.013455999890963235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,5120,0.021596799294153847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,256,0.005285333096981049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,4096,0.011218133568763732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,4096,0.020009599129358926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,3072,128,0.004981333514054617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,3584,0.010597333312034607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,3584,0.019413334131240845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,3584,0.013303466637929282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,10240,0.026349866390228273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,3072,0.009385599692662557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,3072,0.018997333447138467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,2560,0.008633599678675333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,2560,0.01791999936103821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,2560,0.010750933488210043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,2048,0.007574399809042613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,2048,0.017046399911244712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,65536,0.172268803914388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,1536,0.006215466558933258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,1536,0.01641386648019155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,16384,0.04814720153808594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,1024,0.004811733464399974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,1024,0.015660799543062844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,10240,0.03163946668306987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,768,0.004349866509437561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,768,0.015713066856066386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,768,0.00652159998814265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,512,0.003851733356714249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,512,0.01556373337904612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,6144,0.020180267095565797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,256,0.0033941333492596946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,256,0.015026133259137472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,5120,0.018052266041437785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,128,0.003083733220895131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,128,0.014733866850535075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,4096,0.014602667093276978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,64,0.002993066608905792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,64,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2560,32,0.003066666672627131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2560,32,0.014916266997655234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,65536,0.10775360266367595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,3072,0.012266666690508524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,65536,0.08586666584014893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,51200,0.08583573500315347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,2048,0.009290666381518046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,51200,0.07113280296325683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,16384,0.031245867411295574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,51200,0.13584319750467938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,1536,0.00817493349313736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,12288,0.025254400571187337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,16384,0.03271786570549011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,12288,0.028652799129486085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,10240,0.021959465742111207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,10240,0.02581546703974406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,1024,0.0077898666262626644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,8192,0.0178272008895874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,8192,0.023458133141199745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,512,0.005884799857934316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,7168,0.016090666254361473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,7168,0.022886399428049722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,256,0.005235200126965841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,6144,0.014270933469136557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,6144,0.0218666672706604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2560,128,0.005043200155099233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,6144,0.02071253259976705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,5120,0.011995733777681986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,5120,0.02109973430633545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,65536,0.17076160113016764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,4096,0.01034879982471466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,4096,0.019792000452677407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,16384,0.04474773406982422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,3584,0.009462400277455648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,3584,0.018940800428390504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,12288,0.033454934755961105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,3072,0.008839466174443563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,3072,0.01797119975090027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,10240,0.02993706663449605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,2560,0.00788266658782959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,2560,0.01736746629079183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,8192,0.02579200069109599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,2048,0.007032533486684163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,2048,0.016705065965652466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,7168,0.02251519958178202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,1536,0.005649066468079885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,1536,0.016104533274968465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,1536,0.008312533299128216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,1024,0.0045621335506439206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,1024,0.015828266739845276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,5120,0.016982400417327882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,768,0.004194133480389913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,768,0.007231999933719635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,768,0.015447466572125753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,512,0.003681066632270813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,512,0.015390933553377787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,256,0.0032768001159032187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,256,0.014910933375358582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,4096,0.014633599917093912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,128,0.0029696000119050344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,128,0.014709333578745524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,3584,0.013594667116800943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,64,0.0030346666773160298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,64,0.014756266276041666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,2048,32,0.003123199939727783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,3072,0.011943466464678447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,2048,32,0.014724266529083253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,65536,0.08425493240356445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,65536,0.0758453369140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,2560,0.01060479978720347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,51200,0.06744106610616049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,51200,0.06148159901301066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,16384,0.02537493308385213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,16384,0.028698666890462236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,2048,0.009354666868845622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,12288,0.020457599560419717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,12288,0.0254314661026001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,12288,0.028955733776092528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,10240,0.0173962672551473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,1024,0.007614933451016744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,10240,0.023870933055877685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,8192,0.014437333742777506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,8192,0.022743467489878336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,512,0.005971199770768484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,7168,0.013202133774757385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,256,0.005195733408133189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,7168,0.022258132696151733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,6144,0.01206719974676768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,6144,0.022000000874201456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,2048,128,0.004941866795221964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,5120,0.010840533177057902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,5120,0.021588265895843506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,65536,0.14444479942321778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,4096,0.009720533092816671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,51200,0.11151573657989503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,4096,0.01948480010032654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,3584,0.009046399593353271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,16384,0.03978133201599121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,10240,0.026945066452026368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,3072,0.008328533172607422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,3072,0.0187882661819458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,3072,0.010364799698193868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,2560,0.008559999863306682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,2560,0.017614932854970296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,8192,0.022379734118779502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,2048,0.007417599856853485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,2048,0.01725013256072998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,3584,0.019154133399327596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,1536,0.006025599936644236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,1536,0.016607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,7168,0.02087679902712504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,1024,0.005038933455944061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,1024,0.015804800391197204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,6144,0.017476266622543334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,768,0.0044490665197372435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,768,0.015757866700490317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,768,0.006297599772612255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,512,0.003806933263937632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,512,0.015672533710797628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,5120,0.015544533729553223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,256,0.00340693344672521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,256,0.014775466918945313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,4096,0.012387200196584066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,128,0.0030400000512599947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,128,0.014657066265741984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,3584,0.011387733618418376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,2560,0.009889066219329834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,64,0.0030026666820049284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,64,0.014674133062362671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,2048,0.0083146666487058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1536,32,0.003078400095303853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1536,32,0.014662399888038635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,65536,0.05695786476135254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,1536,0.007777066528797149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,65536,0.061755732695261634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,51200,0.045960533618927005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,1024,0.007030400137106578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,51200,0.05208106835683187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,16384,0.017940266927083334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,51200,0.10123093128204345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,16384,0.03180586695671082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,12288,0.013994666934013366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,16384,0.024631466468175253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,10240,0.01227946678797404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,512,0.00559146652619044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,12288,0.022768000761667885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,10240,0.022227199872334798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,8192,0.010792533556620281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,10240,0.024286933739980063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,7168,0.009989333152770997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,8192,0.020286933581034342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,8192,0.020946133136749267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,7168,0.017391999562581383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,6144,0.01018986701965332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,7168,0.020308266083399452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,5120,0.009506133198738099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,6144,0.014084266622861228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,6144,0.0196341335773468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,4096,0.008561066786448161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,5120,0.02099519968032837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,4096,0.010497066378593444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,256,0.00516480008761088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,3584,0.018286933501561485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,3584,0.009705600142478944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,3072,0.007291733225186665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,3072,0.0178272008895874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,3072,0.008903466661771138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,2560,0.0071712002158164975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,2560,0.01713706652323405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,2560,0.008514133095741273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,2048,0.006381866832574208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,2048,0.007565866907437642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,2048,0.01648319959640503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,1536,0.005284266670544943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,1536,0.00698773314555486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,1536,0.015914666652679443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,12288,0.027229867378870648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,1024,0.004546133180459341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,1024,0.015602133671442666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,768,0.004175999760627746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,1024,0.0064074665307998655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,768,0.015482667088508605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,512,0.0037130666275819145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,512,0.015385599931081137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,512,0.005340800185998281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,256,0.003209600100914637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,256,0.014784000317255654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,256,0.004894933104515076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,128,0.0030154667794704436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,128,0.014833066860834757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,64,0.002882133424282074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,4096,0.018902399142583213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,64,0.014532267053922018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,32,0.002951466788848241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,1024,32,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,65536,0.045984001954396565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,65536,0.055731201171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,51200,0.03766080141067505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,65536,0.113156263033549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,51200,0.04640640020370483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,51200,0.08969279925028482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,16384,0.01420799990495046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,16384,0.024142932891845704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,12288,0.011740799744923909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1536,128,0.004780800143877665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,12288,0.021921066443125407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,10240,0.01139520009358724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,12288,0.022859734296798707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,65536,0.12410879929860433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,10240,0.020627200603485107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,10240,0.021512534221013388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,8192,0.010053333640098572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,8192,0.017318399747212727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,7168,0.009576533238093059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,8192,0.01983893314997355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,768,0.00565119981765747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,6144,0.008886399865150451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,7168,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,7168,0.019115734100341796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,5120,0.008280533552169799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,6144,0.02114880084991455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,5120,0.020143999656041463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,5120,0.011534933249155681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,4096,0.009773866335550944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,4096,0.01882986625035604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,3584,0.0073642666141192125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,1024,3584,0.008059733112653096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,3584,0.018388267358144125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,3072,0.007022933165232341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,3072,0.017208532492319743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,3072,0.0090037335952123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,2560,0.007650133470694225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,5120,0.012185600399971009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,2560,0.008109866579373678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,2048,0.007325866818428039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,2048,0.016389333208402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,16384,0.029977599779764812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,1536,0.005076266825199127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,1536,0.01576746702194214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,1536,0.006920533378918965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,1024,0.00415786678592364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,1024,0.015521066387494406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,1024,0.006005333364009857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,768,0.003786666691303253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,768,0.01560533344745636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,4096,0.007118933399518331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,512,0.003504000107447306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,512,0.015294933319091797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,512,0.005308799942334493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,256,0.003221333275238673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,256,0.014778666694959006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,256,0.004986666639645894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,128,0.0029002666473388673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,1024,128,0.00483840008576711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,128,0.004689066608746847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,128,0.014759467045466105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,2560,0.01669973333676656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,64,0.0027893332143624624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,768,32,0.0028512001037597655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,32,0.014784000317255654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,65536,0.03309119939804077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,65536,0.048661335309346514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,65536,0.10002986590067546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,51200,0.027302400271097822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,51200,0.040253865718841556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,51200,0.07678613662719727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,16384,0.021922133366266885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,16384,0.02624853253364563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,12288,0.01015786627928416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,12288,0.020360533396402994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,6144,0.012650666634241739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,10240,0.009272533655166625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,10240,0.021501866976420085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,10240,0.017313067118326822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,8192,0.008353066444396973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,8192,0.019986132780710854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,8192,0.014839466412862143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,768,64,0.014620799819628397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,7168,0.007869866490364075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,6144,0.007162666817506154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,7168,0.013578666249910989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,7168,0.01944533387819926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,5120,0.008169599870840708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,6144,0.012063999970753986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,6144,0.01882986625035604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,4096,0.007099733253320058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,5120,0.010963199536005656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,5120,0.019373865922292073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,3584,0.00681386689345042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,3584,0.01802560091018677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,16384,0.010851200421651204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,3584,0.009300266702969868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,3072,0.0064416001240412395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,3072,0.01752106746037801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,2560,0.006275199850400289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,2048,0.007204266885916392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,2560,0.01683733264605204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,2048,0.005512533088525137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,2048,0.016617600123087564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,2048,0.007107200225194295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,1536,0.004911999901135763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,1536,0.0160480002562205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,1536,0.006693333387374878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,768,0.005612800021966299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,1024,0.004342400034268697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,1024,0.005704533557097117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,1024,0.015566933155059814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,768,0.004011733333269755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,768,0.01530026694138845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,768,0.005529599885145823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,768,3584,0.009398399790128072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,512,0.003607466568549474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,512,0.014921599626541137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,256,0.0032106667757034303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,512,0.0052042668064435325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,256,0.004807466765244802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,128,0.0028917332490285236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,256,0.014587733149528503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,64,0.002845866729815801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,128,0.014692266782124838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,512,32,0.0028245332340399425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,64,0.014365866780281067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,4096,0.018297600746154784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,512,32,0.014723199605941772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,65536,0.021234132846196494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,65536,0.04139626820882161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,65536,0.08774720033009847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,51200,0.01633280018965403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,3072,0.008379733562469483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,51200,0.034406399726867674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,16384,0.00862506628036499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,51200,0.06870933373769125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,16384,0.020889600118001304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,16384,0.02444159984588623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,12288,0.008212266862392426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,12288,0.02026240030924479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,10240,0.019767467180887857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,10240,0.01689066688219706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,8192,0.00812799980243047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,8192,0.018860799074172974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,8192,0.014524799585342408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,7168,0.007529599964618683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,12288,0.02049386699994405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,7168,0.019092265764872232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,7168,0.013296000162760415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,6144,0.007011199990908305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,6144,0.011948800086975098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,5120,0.006600533425807953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,5120,0.018683733542760213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,5120,0.010781866312026978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,128,0.004705066482226053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,4096,0.01820053259531657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,4096,0.009622400005658466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,3584,0.00660693347454071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,3584,0.017745065689086913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,3584,0.009220266342163086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,3072,0.006295466423034668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,4096,0.00955733358860016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,3072,0.008389332890510559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,3072,0.01715946594874064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,2560,0.006044800082842508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,2560,0.016530133287111917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,2560,0.007923200229803721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,2048,0.005427200098832448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,2048,0.016463999946912132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,2048,0.0070933332045873005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,1536,0.00472320020198822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,1536,0.00680320014556249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,1536,0.015625600020090738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,1024,0.004082133372624716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,1024,0.015656532843907674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,1024,0.005718400080998739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,768,0.003736533224582672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,512,2560,0.008020266890525818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,768,0.015502933661142984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,768,0.005403733253479004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,512,0.0033941333492596946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,256,0.003078400095303853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,512,0.005053866902987162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,512,0.015261866648991904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,128,0.0028597332537174227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,6144,0.018689066171646118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,256,0.014575999975204468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,64,0.0027797333896160126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,128,0.004673066735267639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,32,0.0027082666754722597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,64,0.014428800344467163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,65536,0.015530666708946228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,4096,0.006132266422112783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,65536,0.035403732458750406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,65536,0.08731520175933838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,51200,0.031395200888315836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,51200,0.06986026763916016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,16384,0.008376533786455791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,16384,0.024503467480341594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,16384,0.02057066758473714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,12288,0.007575466732184092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,12288,0.01994026700655619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,12288,0.01952000061670939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,10240,0.007008000214894612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,10240,0.01962560017903646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,10240,0.016921599706014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,8192,0.006472533444563548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,8192,0.014702933033307395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,8192,0.018651733795801796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,7168,0.00621973325808843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,7168,0.019086933135986327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,7168,0.01334933340549469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,6144,0.0060362666845321655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,6144,0.018940800428390504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,6144,0.012000000476837159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,5120,0.006389333307743073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,5120,0.010626133282979329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,4096,0.0060479998588562015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,4096,0.018062933286031087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,128,0.01458560029665629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,4096,0.009332266449928284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,3584,0.006450133522351582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,3584,0.017493333419164023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,256,32,0.014654933412869772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,3584,0.009160533547401428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,3072,0.006110933423042297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,3072,0.008210133512814839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,2560,0.005974400043487549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,3072,0.01731200019518534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,51200,0.012882133324941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,2560,0.007842133442560833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,2560,0.016757333278656007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,2048,0.005354666709899902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,1536,0.004680533210436503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,2048,0.00699839989344279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,2048,0.016152532895406087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,1536,0.006783999999364217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,1536,0.01566933294137319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,1024,0.015377066532770791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,768,0.003737599899371465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,768,0.015546666582425437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,256,10240,0.0073173334201176955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,768,0.005465599894523621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,512,0.0033642667035261786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,512,0.015518933534622192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,512,0.0051360001166661584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,256,0.003047466774781545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,256,0.014613333344459533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,256,0.004857600231965383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,128,0.0028885332246621448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,128,0.01463573376337687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,128,0.004655999938646952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,64,0.002679466704527537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,32,0.0026911998788515727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,64,0.014605866869290671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,65536,0.010427733262379963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,32,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,128,5120,0.019040000438690186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,51200,0.009532800316810608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,65536,0.032219733794530234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,16384,0.006714666883150737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,12288,0.00703359991312027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,51200,0.027931733926137285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,16384,0.020296533902486168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,10240,0.006615466872851054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,12288,0.019479467471440633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,8192,0.006274133423964183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,10240,0.019287467002868652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,7168,0.006031999985376993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,8192,0.01872533361117045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,6144,0.005922133227189382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,7168,0.019374932845433554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,5120,0.006247466802597046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,6144,0.01848533352216085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,4096,0.005843199789524078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,5120,0.01913706660270691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,4096,0.01825066606203715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,3072,0.006099199752012888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,3072,0.017402666807174682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,128,1024,0.004045866678158442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,2560,0.006011733412742614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,2048,0.005305600166320801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,128,1024,0.005890133480230967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,2560,0.01652906636397044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,1536,0.004667733112970988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,2048,0.016406400005022685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,1024,0.003945599993069967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,768,0.0035968000690142312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,1536,0.015829333662986757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,1024,0.015708800156911215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,512,0.0032746667663256326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,768,0.015409066279729208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,256,0.0029866665601730345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,512,0.014906666676203408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,128,0.0028575999041398365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,256,0.014705066879590353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,64,0.0026335999369621276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,128,0.014539733529090881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,32,0.0027317332724730173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,64,0.014360533157984415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,32,0.014518400033315023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,65536,0.013294933239618936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,51200,0.01116373340288798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,65536,0.031999999284744264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,16384,0.006675200164318084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,12288,0.007109333574771881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,51200,0.029586132367451983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,16384,0.019832533597946168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,10240,0.00659093310435613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,12288,0.0199178675810496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,10240,0.01943040092786153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,8192,0.006296533346176148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,7168,0.0060928001999855045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,8192,0.019266132513682047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,7168,0.018915200233459474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,5120,0.006168533364931742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,6144,0.018631466229756675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,4096,0.0058229332168896995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,5120,0.01877440015474955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,3584,0.006250666578610738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,3584,0.01766293247540792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,3072,0.005957333246866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,64,3584,0.006391466657320659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,3072,0.01678933302561442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,64,3584,0.01787839929262797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,2560,0.005964800218741099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,256,0.004856533308823904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,2048,0.005310933291912079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,2560,0.016583466529846193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,1536,0.004704000055789947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,1024,0.00395413339138031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,1536,0.01579839984575907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,768,0.0036288000643253326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,1024,0.015543466806411744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,512,0.0033514666060606635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,768,0.015233066678047181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,256,0.0030410667260487873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,512,0.01480959951877594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,128,0.002807466685771942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,97,256,12288,0.019474132855733236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,64,0.002700799951950709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,128,0.014425599575042724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,32,0.0026464000344276427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,64,0.014607999722162882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,32,0.014498133460680643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,97,32,6144,0.0058335999647776285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,4096,0.018175999323527016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,2048,0.016225066781044007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,97,32,256,0.014661332964897156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,16384,0.40418027242024734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,16384,0.760093879699707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,16384,0.3714026769002279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,12288,0.3096501350402832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,12288,0.5779477437337239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,12288,0.3090943972269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,51200,1.1322069803873698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,51200,1.2113557179768881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,10240,0.48494612375895185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,10240,0.26063359578450523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,10240,0.24995627403259277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,8192,0.2211712042490641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,8192,0.2445674737294515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,7168,0.33892800013224283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,7168,0.18870186805725098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,7168,0.17076053619384765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,51200,2.371001688639323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,5120,0.141101868947347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,6144,0.16506346066792804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,5120,0.24530773162841796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,6144,0.2936842600504557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,4096,0.11694719791412353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,5120,0.14293546676635743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,3584,0.17478826840718586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,3584,0.1058517297108968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,3584,0.09310293197631836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,3072,0.09909013112386068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,3072,0.17385493914286296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,2560,0.1270133336385091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,8192,0.38534294764200844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,2560,0.08083199659983317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,2560,0.07770240306854248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,2048,0.10414186318715413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,2048,0.058166400591532386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,2048,0.07612693309783936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,1536,0.05657386779785156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,1536,0.09077226320902507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,1536,0.04657599925994873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,1024,0.056075731913248696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,768,0.04394773244857788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,1024,0.03780373334884644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,1024,0.04820693333943685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,768,0.037571199735005695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,512,0.032076799869537355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,512,0.0302784005800883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,512,0.02344320019086202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,256,0.01837013363838196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,128,0.010546132922172546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,256,0.022992000977198283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,128,0.015871999661127727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,128,0.020438400904337566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,64,0.008154666423797608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,6144,0.15337279637654622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,32,0.007389866809050243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,64,0.018554667631785073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,32,0.018617600202560425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,4096,0.1994655927022298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,3072,0.08203840255737305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,768,0.030734932422637938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,256,0.01795733372370402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,4096,0.10358613332112629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,16384,0.6059808095296224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,65536,1.22673708597819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,51200,0.9640533447265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,65536,1.1893472035725912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,16384,0.32304747899373376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,51200,1.8927103678385417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,12288,0.48258879979451497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,12288,0.24970986048380533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,12288,0.26288960774739584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,65536,2.369453938802083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,10240,0.3874378522237142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,8192,0.17367040316263835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,10240,0.23927787144978843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,8192,0.30231466293334963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,8192,0.17738240559895832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,7168,0.2653183937072754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,7168,0.15695892969767253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,6144,0.23031039237976075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,6144,0.15114986101786296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,5120,0.19399466514587402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,5120,0.11608533064524333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,5120,0.10182399749755859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,4096,0.16155200004577636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,4096,0.09687466621398926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,3584,0.13820692698160808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,3584,0.08729813098907471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,6144,0.12426986694335937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,3072,0.12372480233510334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,16384,0.30316801071166993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,3072,0.07815360228220622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,51200,0.9351989110310873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,2560,0.10094506740570068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,3072,0.07253333727518717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,2560,0.05666559934616089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,2048,0.08267306486765544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,2560,0.07503893375396728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,2048,0.05873279968897501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,1536,0.0489738663037618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,10240,0.21952640215555824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,1024,0.05190186500549317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,1024,0.03882773319880168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,7168,0.13861865997314454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,768,0.039854931831359866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,768,0.033215999603271484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,512,0.02879253427187602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,512,0.027189334233601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,1536,0.07275626659393311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,4096,0.08336213429768881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,256,0.014380799730618796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,256,0.022846933205922446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,128,0.009810133775075277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,3584,0.07447360356648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,128,0.019723733266194664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,64,0.0074976002176602675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,64,0.018911999464035035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,51200,32,0.006989866495132446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,51200,32,0.01899519960085551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,2048,0.04649600187937419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,65536,0.7781450907389323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,1536,0.037759999434153244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,65536,0.3982207934061686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,1024,0.028680533170700073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,65536,0.3802538553873698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,768,0.02440426747004191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,51200,0.606497065226237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,16384,0.20010879834493003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,51200,0.3146122614542643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,16384,0.114246400197347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,51200,0.2938912073771159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,16384,0.11809066931406657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,512,0.018972800175348917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,12288,0.14321494102478027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,12288,0.07845333417256674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,10240,0.11947946548461914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,12288,0.09203519821166992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,10240,0.07856213251749675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,256,0.01455573340257009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,10240,0.0661898652712504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,8192,0.06654080152511596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,7168,0.08547733624776205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,7168,0.04956800142923991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,51200,128,0.012257066369056702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,6144,0.07439786593119303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,6144,0.043398400147755936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,6144,0.054370133082071934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,5120,0.04792533318201701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,4096,0.051412268479665124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,5120,0.03792746861775716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,4096,0.0418069322903951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,4096,0.031914667288462324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,3584,0.04554986556371053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,3584,0.029063467184702558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,3072,0.04002346595128377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,3072,0.025919999678929644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,2560,0.0340554674466451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,2560,0.0323360006014506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,2560,0.022921599944432578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,2048,0.02827626665433248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,2048,0.028380799293518066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,2048,0.01977919936180115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,1536,0.02248959938685099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,1536,0.024950400988260905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,1536,0.015995732943216958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,1024,0.015731199582417806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,1024,0.021635200579961142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,1024,0.01272213359673818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,768,0.012272000312805176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,768,0.020126932859420778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,768,0.011458133657773335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,512,0.009251200159390767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,512,0.009805867075920105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,256,0.006448000172773997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,256,0.015955199797948204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,256,0.007361066838105519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,128,0.004223999877770742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,8192,0.1022602637608846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,128,0.01569919983545939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,7168,0.05993173519770304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,64,0.0039381332695484165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,5120,0.06266346772511801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,64,0.015898666779200234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,32,0.004331733286380768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,32,0.016055466731389363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,3584,0.038736001650492353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,3072,0.0356874664624532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,65536,0.32431252797444665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,65536,0.2998485247294108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,65536,0.5711061477661132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,51200,0.4506805419921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,512,0.01859626571337382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,16384,0.07964906692504883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,16384,0.18083732922871906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,16384,0.09452160199483237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,51200,0.2613215923309326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,51200,0.23620905876159667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,12288,0.12203946908315022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,12288,0.07224000295003255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,10240,0.10355733235677082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,10240,0.07111466725667318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,10240,0.05252906481424967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,8192,0.07260053157806397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,8192,0.06252906719843546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,8192,0.05497920115788778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,8192,0.04380160172780355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,7168,0.04887679815292358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,7168,0.06567360162734985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,6144,0.057843200365702306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,6144,0.03470826546351115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,6144,0.04589759906133016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,5120,0.04910293420155843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,5120,0.029816534121831256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,5120,0.04012480179468791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,4096,0.03602026700973511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,4096,0.02553066611289978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,3584,0.03468586603800456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,3584,0.0336736003557841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,3072,0.030740267038345336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,3072,0.031173332532246904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,2560,0.02638186613718669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,2560,0.027937066555023194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,2560,0.01812160015106201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,2048,0.022012799978256226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,2048,0.02529279987017314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,2048,0.01535360018412272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,1536,0.017564799388249716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,1536,0.022776534159978233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,7168,0.039185067017873124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,1024,0.012119467059771221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,4096,0.039282135168711346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,128,0.007006933291753133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,1024,0.010748799641927083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,1024,0.02049493392308553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,768,0.010082133611043294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,3584,0.023037866751352946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,768,0.009566932916641235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,768,0.01922773321469625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,512,0.007454933226108551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,3072,0.020538665850957236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,256,0.00452693353096644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,512,0.016833066940307617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,128,0.003815466662247976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,256,0.006478933493296306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,128,0.01520853340625763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,64,0.003568000098069509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,32,0.003885866701602936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,64,0.01558080017566681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,32,0.015757866700490317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,65536,0.2640543937683105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,65536,0.49448318481445314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,12288,0.06219413280487061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,512,0.007843199868996937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,51200,0.22858880360921224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,51200,0.39058561325073243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,256,0.01585706671079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,16384,0.1371946652730306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,16384,0.07908906936645507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,16384,0.078548264503479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,12288,0.10594240029652913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,12288,0.06401919921239217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,12288,0.05514026482899984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,10240,0.07788373629252116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,10240,0.05658133427302042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,8192,0.0631989320119222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,10240,0.047133866945902506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,8192,0.03830506801605225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,8192,0.04849173227945964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,7168,0.0549237330754598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,7168,0.044622933864593504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,7168,0.03387519915898641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,6144,0.030024532477060956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,6144,0.04071466525395711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,5120,0.04066559871037801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,5120,0.026220800479253133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,5120,0.03674986759821574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,65536,0.2852661450703939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,4096,0.033771733442942306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,4096,0.021948800484339396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,4096,0.03224640091260274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,3584,0.034082134564717606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,3584,0.030261333783467608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,1536,0.012962133685747782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,3072,0.02677546739578247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,3584,0.02006933291753133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,2560,0.02291946609814962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,3072,0.02841813365618388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,2560,0.025201066335042314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,2048,0.019233065843582153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,2560,0.015786666671435037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,1536,0.014820266763369241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,2048,0.013674666484196981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,1536,0.011545600493748982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,1536,0.021581866343816123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,1024,0.010940800110499065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,51200,0.21206293106079102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,1024,0.02012053330739339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,1024,0.00972160001595815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,768,0.009036800265312195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,768,0.007995733122030894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,768,0.01846933364868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,512,0.007335466643174489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,512,0.007014399766921997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,512,0.016250666975975037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,256,0.015820800264676412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,128,0.0038592000802357995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,128,0.01527466674645742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,128,0.005558399856090546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,64,0.0035242666800816857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,128,0.005836800237496694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,64,0.015737600127855935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,32,0.003667200108369192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,32,0.015365333358446757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,6144,0.04831039905548096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,3072,0.017990400393803917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,65536,0.2269183953603109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,65536,0.4112853368123372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,2048,0.02327893376350403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,65536,0.22783573468526205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,16384,0.10546560287475586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,51200,0.18256319363911946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,256,0.004342400034268697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,51200,0.18011946678161622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,51200,0.3184170722961426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,12288,0.07355946699778239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,12288,0.0550986647605896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,10240,0.07160212993621826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,10240,0.040793601671854654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,10240,0.04895573457082113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,8192,0.0501749316851298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,8192,0.041919998327891034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,7168,0.04456640084584554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,7168,0.038585599263509116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,7168,0.029516800244649248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,6144,0.03950613339742025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,6144,0.03548373381296794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,5120,0.033351465066274004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,5120,0.032263465722401935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,16384,0.0670570691426595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,4096,0.027361067136128743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,4096,0.02818133234977722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,3584,0.02440746625264486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,3584,0.026238934199015303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,256,0.005966933568318685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,3072,0.021676800648371377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,3072,0.024330667654673257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,16384,0.0615775982538859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,12288,0.0475221316019694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,2560,0.018578133980433145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,8192,0.03283093372980754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,2560,0.02277759909629822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,2048,0.015147733688354491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,2048,0.021221333742141725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,1536,0.011811199784278869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,6144,0.026526933908462523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,1536,0.01978879968325297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,1024,0.008954667051633199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,1024,0.01839039921760559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,1024,0.008905599514643352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,5120,0.023005867004394533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,768,0.0075914666056633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,768,0.007796266674995422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,768,0.016921599706014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,512,0.005860266586144766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,512,0.016246400276819863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,256,0.004211199780305227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,256,0.015966932972272238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,4096,0.019598933060963948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,128,0.0035797332723935447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,128,0.015176533659299215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,3584,0.018125865856806436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,64,0.0034474665919939675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,64,0.015498666961987814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,32,0.0037952000896135964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,32,0.015397333105405173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,3072,0.01606186628341675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,65536,0.21335466702779135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,65536,0.35455252329508463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,51200,0.28035411834716795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,65536,0.225218137105306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,51200,0.16341226895650227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,2048,0.01262933313846588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,2560,0.014509866635004679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,16384,0.09252266883850098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,1536,0.01100373367468516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,16384,0.06084693272908529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,16384,0.061869867642720545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,12288,0.06501973470052083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,10240,0.05501333475112915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,12288,0.047147734959920244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,10240,0.04101119836171468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,10240,0.044733866055806475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,8192,0.04527573188145955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,8192,0.038840532302856445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,256,0.006285866598288219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,7168,0.03945813179016113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,7168,0.03593279918034871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,6144,0.03516586621602376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,6144,0.03333866596221924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,128,0.00572266678015391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,5120,0.02943893273671468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,5120,0.03009066581726074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,12288,0.051351467768351235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,4096,0.024609067042668662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,4096,0.02658560077349345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,512,0.007020799815654755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,4096,0.018920532862345376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,3584,0.021824000279108684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,3584,0.025179733832677204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,3072,0.019449599583943687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,3072,0.023571199178695677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,51200,0.18078719774882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,2560,0.018709333737691243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,2560,0.021922133366266885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,2048,0.013499733805656434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,2048,0.02068906625111898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,2048,0.011707733074824016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,1536,0.010873599847157796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,1536,0.01954453388849894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,7168,0.029471999406814574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,1024,0.008347733815511068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,1024,0.01679253379503886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,1024,0.007961600025494894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,768,0.007162666817506154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,6144,0.025916800896326704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,768,0.015797332922617594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,8192,0.03297280073165894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,768,0.007275733351707459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,512,0.004693333307902018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,256,0.003918933371702829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,512,0.015897599856058757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,256,0.015432533621788026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,128,0.0034741332133611047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,128,0.01532906691233317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,5120,0.02267199953397115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,64,0.003419733295838038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,64,0.015424000223477683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,32,0.0034933333595593774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,32,0.01537493367989858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,3584,0.016977065801620485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,3072,0.01539306640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,65536,0.2966496149698893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,65536,0.18509546915690106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,65536,0.21286187171936036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,51200,0.23557653427124023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,51200,0.14952319463094074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,16384,0.09611946741739909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,16384,0.055776000022888184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,12288,0.0636298656463623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,2560,0.013741866747538248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,12288,0.04647253354390462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,12288,0.044647467136383054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,10240,0.054839468002319335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,10240,0.04150613149007161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,8192,0.03885013262430827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,8192,0.0356768012046814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,1536,0.010258133212725323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,7168,0.034618667761484784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,7168,0.033166933059692386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,7168,0.02798186739285787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,6144,0.030743465820948286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,512,0.00647680014371872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,256,0.005645866692066193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,128,0.005314133564631144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,5120,0.026241066058476765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,5120,0.028282666206359865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,5120,0.020746666193008422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,4096,0.021802665789922078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,4096,0.017871999740600587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,3584,0.019406932592391967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,3584,0.023439999421437582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,6144,0.030748800436655684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,3072,0.017173333962758382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,3072,0.022822399934132896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,3072,0.014838400483131408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,2560,0.014532267053922018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,2560,0.021130667130152384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,51200,0.17039252916971842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,2560,0.013176533579826354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,2048,0.011953066786130269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,4096,0.02578666607538859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,2048,0.020150399208068846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,1536,0.009898666540781658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,1536,0.01930346687634786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,16384,0.05724373261133829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,1024,0.007997866471608479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,1024,0.016942934195200602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,10240,0.03840106725692749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,768,0.006832000116507213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,768,0.015955199797948204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,768,0.00695360004901886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,512,0.004499199986457825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,8192,0.030851199229558306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,512,0.015735466281572977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,512,0.006145066519578298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,256,0.003902933249870936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,256,0.015364266435305276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,128,0.0034122665723164878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,6144,0.024765866994857787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,128,0.015130666891733804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,64,0.0032960000137488045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,64,0.015125333269437154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,32,0.003365333378314972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,32,0.015268266201019287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,3584,0.0163263996442159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,65536,0.2434293270111084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,65536,0.15781653722127278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,65536,0.2059552033742269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,51200,0.1918794631958008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,51200,0.12756266593933105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,16384,0.06998079617818197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,16384,0.05638186534245809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,2048,0.011363200346628825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,12288,0.0552021344502767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,12288,0.04266133308410645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,12288,0.04220373233159383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,1536,0.009880533814430237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,10240,0.048138666152954104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,10240,0.03781973520914714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,1024,0.007843199868996937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,8192,0.029733333985010785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,8192,0.04062399864196777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,7168,0.029803733030954998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,7168,0.030668799082438154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,7168,0.027004800240198773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,6144,0.026441599925359088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,6144,0.028037333488464357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,6144,0.02379306753476461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,5120,0.022702932357788086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,5120,0.025448532899220784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,4096,0.018556799491246542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,256,0.005489066739877065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,4096,0.023244800170262654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,3584,0.01676266590754191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,128,0.005211733281612396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,3584,0.022171733776728313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,3072,0.014777599771817525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,3072,0.02144533395767212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,8192,0.03287039995193482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,2560,0.012599466244379678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,2560,0.020627200603485107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,2560,0.012897066275278726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,2048,0.010703999797503154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,2048,0.019436800479888917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,2048,0.010844799876213073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,1536,0.008780800302823384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,1536,0.017777067422866822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,4096,0.016884267330169678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,1024,0.007014399766921997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,1024,0.016058666507403056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,51200,0.16310933430989583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,768,0.005795200169086456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,768,0.016124799847602844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,16384,0.05446933507919312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,512,0.00401706670721372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,512,0.015638400117556253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,10240,0.036957867940266925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,256,0.003655466685692469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,256,0.015501866738001505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,5120,0.020107734203338622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,128,0.003289599965016047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,128,0.014942933122316995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,3584,0.015588266650835672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,64,0.0032149332265059153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,64,0.01507306694984436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,32,0.0032032000521818793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,3072,0.014332800110181173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,32,0.015180800358454385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,65536,0.2001802603403727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,65536,0.13490452766418456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,51200,0.15724159876505533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,65536,0.19406506220499675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,51200,0.10927573045094807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,1536,0.00951573352018992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,16384,0.054807468255360925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,16384,0.04659626483917236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,1024,0.0076341331005096436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,12288,0.04358079830805461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,12288,0.039933868249257404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,768,0.006857599814732869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,10240,0.03767253160476684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,10240,0.0336458683013916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,10240,0.032398933172225954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,8192,0.03155093391736348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,512,0.006095999975999197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,8192,0.02940373420715332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,7168,0.02873813311258952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,7168,0.027046400308609008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,256,0.005538133283456167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,6144,0.025914667050043742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,6144,0.025356799364089966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,6144,0.021117866039276123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,5120,0.018780799706776936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,5120,0.023654399315516154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,128,0.0052821333209673565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,4096,0.015396266182263692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,4096,0.022184532880783082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,4096,0.015867732961972556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,3584,0.013670399785041809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,3584,0.021537067492802937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,16384,0.04860053459803264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,3072,0.012124799688657125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,3072,0.02038933237393697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,12288,0.03723306655883789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,2560,0.010712533195813497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,2560,0.019735467433929444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,2560,0.011915733416875202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,2048,0.009282132983207703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,2048,0.018329600493113198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,1536,0.007787733276685078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,7168,0.023891200621922813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,1536,0.016177067160606386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,1024,0.006234666705131531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,1024,0.016241066654523215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,51200,0.15122666358947753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,768,0.004324266811211904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,768,0.01595626672108968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,5120,0.017968000968297322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,768,0.006742399930953979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,512,0.0038581334054470064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,512,0.01565439999103546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,256,0.003491200009981791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,256,0.014833066860834757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,3584,0.014658133188883463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,8192,0.026514132817586262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,128,0.0032138665517171226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,128,0.014756266276041666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,64,0.0030986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,64,0.015011200308799743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,32,0.003419733295838038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,32,0.015280000368754067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,65536,0.1810186703999837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,65536,0.12790079911549884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,3072,0.013491200407346091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,65536,0.1647808074951172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,51200,0.1441813309987386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,51200,0.10484373569488525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,16384,0.04891200065612793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,2048,0.010392533739407857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,16384,0.04366506735483806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,12288,0.03888746500015259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,1536,0.008917333682378133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,12288,0.03648639917373657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,10240,0.03364053169886271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,1024,0.0071487997968991595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,10240,0.03170773386955261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,8192,0.028398933013280232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,10240,0.03116053342819214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,8192,0.027430399258931475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,7168,0.023669334252675374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,7168,0.025534933805465697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,512,0.00600853314002355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,6144,0.021062399943669638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,6144,0.02481493353843689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,256,0.0054848000407218935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,5120,0.01680213411649068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,5120,0.02267199953397115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,128,0.005102933446566264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,4096,0.013692800203959146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,4096,0.021564799547195434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,51200,0.1334005355834961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,3584,0.012932266791661581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,3584,0.020668800671895346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,16384,0.0457152009010315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,3072,0.011180800199508668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,12288,0.03640640179316203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,3072,0.012918399771054587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,3072,0.020286933581034342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,2560,0.009830400347709656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,2560,0.01937920053799947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,2048,0.00851200024286906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,2048,0.018106667200724284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,8192,0.025923200448354083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,1536,0.007419733206431071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,1536,0.016127999623616537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,1536,0.008326399823029835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,1024,0.005662933488686879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,7168,0.02334933280944824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,1024,0.016005333264668783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,768,0.004249600072701773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,768,0.015980799992879234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,6144,0.020913066466649373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,768,0.006541866560777028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,512,0.003832533210515976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,512,0.015759999553362526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,512,0.005960533519585928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,256,0.0034645333886146545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,256,0.015054933230082192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,5120,0.01774079998334249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,128,0.003234133372704188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,4096,0.015250133474667868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,128,0.014456533392270408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,64,0.0030346666773160298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,64,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,32,0.0032106667757034303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,32,0.01474453310171763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,65536,0.15889066060384113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,65536,0.11471359729766846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,3584,0.014430933197339377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,51200,0.12516266504923504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,2560,0.011534933249155681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,51200,0.09275840123494467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,51200,0.15007680257161457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,16384,0.046538666884104414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,2048,0.010030933221181234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,16384,0.04027093251546224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,12288,0.03457599878311157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,12288,0.0345909317334493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,12288,0.03744106690088908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,10240,0.029789867003758748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,10240,0.0302784005800883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,10240,0.03318293293317159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,8192,0.02517333428064982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,8192,0.026786132653554277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,7168,0.021746132771174112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,7168,0.02453546722730001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,6144,0.019401599963506065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,6144,0.023818665742874147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,6144,0.02105493346850077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,5120,0.01626240015029907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,5120,0.022372267643610635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,5120,0.018605866034825645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,4096,0.013422933220863343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,4096,0.020777599016825358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,128,0.005191466708978017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,3584,0.011363200346628825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,16384,0.04874133268992106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,3584,0.019992534319559732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,3072,0.010381866494814556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,1024,0.006988800068696339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,3072,0.019359999895095827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,2560,0.009305600325266521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,2560,0.018473599354426065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,2560,0.011736533045768738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,2048,0.008107733229796093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,2048,0.017054933309555053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,256,0.00544106662273407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,1536,0.006871466835339864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,1536,0.016229333480199178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,65536,0.19424427350362142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,1024,0.0047647997736930845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,1024,0.01630506714185079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,8192,0.027126399676005046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,768,0.004186666508515676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,768,0.015924266974131265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,7168,0.024177066485087075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,512,0.0037813333173592886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,512,0.015562666455904641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,4096,0.01557866632938385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,256,0.003435733417669932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,3584,0.01427839994430542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,256,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,128,0.0032266666491826378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,256,0.005364266534646353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,128,0.01497066617012024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,3072,0.013119999567667642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,64,0.0030239999294281008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,32,0.0031498665610949195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,64,0.015056000153223673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,32,0.01499626636505127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,65536,0.137007999420166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,65536,0.10066453615824382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,2048,0.009966933727264404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,51200,0.07937173048655191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,51200,0.1099402666091919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,16384,0.037088000774383546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,1536,0.008501332998275758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,16384,0.04527466694513957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,12288,0.029781333605448407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,12288,0.031318400303522745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,1024,0.00717439999183019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,12288,0.03515093326568604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,10240,0.02567039926846822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,10240,0.02872640093167623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,8192,0.021397332350413002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,10240,0.030873600641886396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,8192,0.025709867477416992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,7168,0.01943999926249186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,7168,0.024213333924611412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,7168,0.02299519975980123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,6144,0.01541759967803955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,768,0.006538666784763336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,512,0.00591786652803421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,6144,0.02352213263511658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,5120,0.013399466872215271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,16384,0.03654079834620158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,5120,0.02175359924634298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,4096,0.0124917338291804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,4096,0.020119466384251914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,128,0.005013333261013031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,3584,0.011464533209800721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,3584,0.019760000705718993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,65536,0.16633706092834472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,3072,0.009371733665466309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,3072,0.01898026665051778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,51200,0.1348298708597819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,2560,0.00853653351465861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,2560,0.017044266064961754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,8192,0.026210133234659833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,2048,0.007386666536331177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,2048,0.016921599706014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,1536,0.006525866687297821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,1536,0.016226133704185484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,6144,0.02017813324928284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,1024,0.00458133320013682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,1024,0.015770666797955833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,5120,0.017062399784723917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,768,0.004267733295758565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,768,0.015650133291880287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,4096,0.014334932963053385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,512,0.003802666564782461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,512,0.015268266201019287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,512,0.00586346685886383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,256,0.003437866767247518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,256,0.014940800269444785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,256,0.005302399893601735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,128,0.0030975999931494398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,128,0.01476800044377645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,3584,0.013179733355840047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,64,0.0029343999922275543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,64,0.014738133549690247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,32,0.0030464000999927522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,32,0.014833066860834757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,65536,0.1067242701848348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,65536,0.08509653409322103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,3072,0.011941333611806233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,51200,0.08483093579610189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,2560,0.010823466380437215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,65536,0.16461973190307616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,16384,0.030777599414189654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,51200,0.0699285348256429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,51200,0.13096853097279865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,16384,0.033062400420506795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,16384,0.04205013513565063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,12288,0.028231465816497804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,12288,0.03315093318621318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,10240,0.02177066604296366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,10240,0.026100265979766845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,8192,0.017822933197021485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,8192,0.024344533681869507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,2048,0.008999466896057129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,7168,0.01581653356552124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,7168,0.022593067089716593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,1536,0.007972266773382823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,6144,0.014241066575050355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,6144,0.021800533930460612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,1024,0.006886399785677592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,5120,0.012959999839464822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,5120,0.020966400702794395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,768,0.006347733239332835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,4096,0.0107722669839859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,5120,0.016756266355514526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,4096,0.014434132973353067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,4096,0.01973973313967387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,3584,0.009916800260543823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,3584,0.013357866803805032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,3584,0.01922986706097921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,3072,0.008968533078829447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,3072,0.01797653237978617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,3072,0.011846400300661723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,2560,0.00823466678460439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,2560,0.017171200116475424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,2560,0.01048533320426941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,2048,0.007474133372306823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,2048,0.016923733552296958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,128,0.0050570666790008545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,1536,0.0058677335580190025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,1536,0.016103466351826988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,1024,0.004572799801826477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,1024,0.01571626663208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,1024,0.0068896000583966565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,768,0.004056533426046371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,768,0.015646933515866598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,12288,0.025029333432515462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,512,0.0037418665985266366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,512,0.015408000349998474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,10240,0.029174399375915528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,256,0.0033674667278925574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,256,0.01483626663684845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,256,0.005233066777388254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,8192,0.023881600300470988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,128,0.003020799905061722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,128,0.01467626690864563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,7168,0.021647999684015908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,64,0.0029546665648619336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,6144,0.01909866730372111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,32,0.0030432000756263735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,64,0.01480959951877594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,32,0.014856533209482829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,65536,0.083133864402771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,65536,0.07594772974650064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,51200,0.06724692980448405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,51200,0.06107626756032308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,51200,0.11270506381988525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,16384,0.025041067600250246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,2048,0.009289600451787313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,16384,0.02846933404604594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,12288,0.019744000832239785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,1536,0.00817920019229253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,12288,0.027012266715367633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,10240,0.01731733282407125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,10240,0.02502080003420512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,10240,0.025913600126902265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,8192,0.014243200421333313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,768,0.006345599889755249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,8192,0.02288320064544678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,7168,0.013883733749389648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,7168,0.022284799814224245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,7168,0.01987733244895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,6144,0.01241919994354248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,6144,0.02143146594365438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,6144,0.017478400468826295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,5120,0.011020800471305848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,5120,0.020154666900634766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,512,0.005757866799831391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,4096,0.009578667084376017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,4096,0.018219733238220216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,128,0.004954666892687479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,3584,0.009036800265312195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,3584,0.018119466304779053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,3072,0.008250666658083598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,3072,0.01753066579500834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,65536,0.14552213350931803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,2560,0.007118933399518331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,2560,0.016963199774424235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,2560,0.009527466694513957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,2048,0.006159999966621399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,2048,0.016660267114639284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,2048,0.008363733688990276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,16384,0.038123734792073566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,1536,0.00521066685517629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,1536,0.01639253298441569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,1024,0.004494933287302653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,12288,0.029711999495824176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,1024,0.01565439999103546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,768,0.004126933217048645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,768,0.015621333320935567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,8192,0.022083199024200438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,512,0.0037439999481042228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,512,0.015291733543078103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,512,0.005524266759554545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,256,0.0032842665910720824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,256,0.014966400464375815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,256,0.005017599960168203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,128,0.0029696000119050344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,128,0.014734933773676554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,128,0.004867200056711833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,5120,0.015131733814875283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,64,0.002845866729815801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,64,0.014819199840227762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,32,0.002942933390537898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,32,0.014817066987355552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,65536,0.05639359951019287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,4096,0.01232319970925649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,65536,0.06128426790237427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,65536,0.11907093524932862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,51200,0.045550934473673504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,51200,0.051830399036407473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,16384,0.018657066424687705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,51200,0.09716800053914389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,16384,0.026183466116587322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,12288,0.014536533753077188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,12288,0.02362346649169922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,12288,0.024692267179489136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,10240,0.01365013321240743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,10240,0.022377600272496544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,10240,0.021997867027918498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,8192,0.012160000205039979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,8192,0.021086933215459187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,8192,0.018336000045140584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,7168,0.011383466919263204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,3584,0.011609599987665812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,7168,0.01617173353830973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,7168,0.021843200922012328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,6144,0.010819199681282043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,6144,0.013987200458844504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,5120,0.009825066725413004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,6144,0.02113599975903829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,5120,0.012329600254694621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,5120,0.01978879968325297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,4096,0.018321067094802856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,1536,0.007558399935563405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,3584,0.008469333251317341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,1024,0.00625493327776591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,3584,0.01801919937133789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,3584,0.009727999567985535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,3072,0.0077344000339508055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,3072,0.017373865842819212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,2560,0.006235733131567637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,2560,0.01686186591784159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,2560,0.008235733211040496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,4096,0.008685866991678875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,3072,0.01035520037015279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,2048,0.005559466779232025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,1536,0.004902400076389313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,2048,0.007524266839027405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,2048,0.016428800423940022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,1024,0.00421973317861557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,1536,0.016088533401489257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,1024,0.015801599621772765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,768,0.003917866696914038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,768,0.015343999862670899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,768,0.005648000041643778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,512,0.003519999980926514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,512,0.015037866433461508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,512,0.005377066632111868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,256,0.0031583999594052637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,256,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,256,0.004946133494377137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,128,0.002932266642649968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,128,0.014699733257293702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,128,0.004808533191680908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,64,0.002855466554562251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,64,0.01458560029665629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,32,0.0028959999481836954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,16384,0.031223465998967487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,32,0.014504533012708029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,65536,0.045324798425038657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,65536,0.05551679929097494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,51200,0.037462401390075686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,51200,0.047042131423950195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,51200,0.0841439962387085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,4096,0.010140800476074218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,16384,0.016745599110921223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,16384,0.02618880073229472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,16384,0.029098665714263915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,12288,0.012195199728012085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,12288,0.022664533058802287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,12288,0.02280319929122925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,10240,0.01125440001487732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,10240,0.02178986668586731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,8192,0.01969919999440511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,8192,0.01591679950555166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,7168,0.010384000341097514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,7168,0.019421867529551187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,7168,0.014009599884351095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,6144,0.009687466422716777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,6144,0.019397334257761637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,6144,0.01248426636060079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,5120,0.008809600273768108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,5120,0.019181867440541588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,5120,0.01099733312924703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,4096,0.007495466868082683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,3072,0.008689066767692566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,4096,0.018179200092951455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,4096,0.009645866354306538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,3584,0.007062399884064992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,3584,0.009322667121887207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,3072,0.006567466755708058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,3072,0.018260266383488974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,1536,0.006862933437029521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,2560,0.006242133180300395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,1024,0.005972266693909963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,2560,0.007997866471608479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,2560,0.016873600085576375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,2048,0.00555733342965444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,10240,0.020259199539820354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,2048,0.007096533477306366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,1536,0.00489279975493749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,1536,0.01593280037244161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,1536,0.006788266698519389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,1024,0.004120533416668574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,1024,0.015710933009783427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,1024,0.0058218667904535925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,768,0.003824000060558319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,768,0.015235199530919393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,768,0.005579733351866404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,512,0.0035071998834609987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,512,0.015507200360298156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,512,0.005278933544953664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,256,0.003234133372704188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,256,0.014590932925542196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,256,0.0049450665712356566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,128,0.002959999938805898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,768,0.005916800101598104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,128,0.014455466469128927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,64,0.002841600030660629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,64,0.014570666352907815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,32,0.002794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,65536,0.0336949348449707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,65536,0.04922773440678914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,3584,0.018023467063903807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,65536,0.10764799912770588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,65536,0.08867093722025553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,3072,0.008416000008583068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,51200,0.0279968003431956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,51200,0.04083306789398193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,16384,0.012475732962290447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,51200,0.07439573605855307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,16384,0.026152533292770386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,16384,0.02232746680577596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,12288,0.010667733351389567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,8192,0.011080533266067505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,2048,0.01639893352985382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,10240,0.01016426682472229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,12288,0.020152533054351808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,8192,0.00918506681919098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,10240,0.02038933237393697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,8192,0.019495467344919838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,7168,0.00836906631787618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,7168,0.019358932971954346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,6144,0.007639466722806294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,6144,0.018506666024525963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,6144,0.012239999572436015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,5120,0.00690773328145345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,5120,0.019061332941055296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,5120,0.01069546639919281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,32,0.014494933684666953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,4096,0.0064416001240412395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,4096,0.018172800540924072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,3584,0.006748799979686737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,3584,0.017489065726598106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,3584,0.009306666254997254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,3072,0.006411733229955037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,3072,0.017594667275746663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,3072,0.008291199803352356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,2560,0.006278400123119354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,2560,0.01688106656074524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,2560,0.007965866724650066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,2048,0.005599999924500784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,12288,0.022075732549031578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,2048,0.01622719963391622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,8192,0.014755200346310934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,1536,0.004894933104515076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,1536,0.015867732961972556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,1024,0.004186666508515676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,1024,0.015620266397794088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,1024,0.005760000149408976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,768,0.0038304001092910765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,768,0.015599999825159708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,768,0.005357866485913595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,512,0.0035648000737031303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,512,0.015278933445612588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,128,0.004680533210436503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,512,0.005241600175698599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,256,0.003068800022204717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,256,0.004874666531880697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,256,0.014640000462532044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,128,0.0029535998900731405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,128,0.004609066744645437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,128,0.014603733023007711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,64,0.002721066772937775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,32,0.0028213332096735638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,64,0.014452266693115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,32,0.01453013320763906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,65536,0.02219840089480082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,65536,0.03970239957173665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,65536,0.08735679785410563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,51200,0.017054933309555053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,51200,0.03258453408877055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,16384,0.00937493344148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,51200,0.06827413241068522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,2048,0.006980266670385997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,12288,0.008123733103275299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,16384,0.020312533775965372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,10240,0.007130666573842366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,12288,0.019796266158421835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,1536,0.006667733192443848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,12288,0.019462400674819948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,10240,0.01979093352953593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,10240,0.01688533425331116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,7168,0.006482133269309997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,10240,0.017246933778127034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,8192,0.014425599575042724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,8192,0.019037866592407228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,6144,0.006159999966621399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,7168,0.01344319979349772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,7168,0.019406932592391967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,5120,0.006609066824118296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,6144,0.011896533767382304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,6144,0.01864746610323588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,5120,0.018819200992584228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,4096,0.006181333462397257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,5120,0.010637866457303365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,7168,0.013545599579811097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,4096,0.017850667238235474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,4096,0.009609599908192951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,3584,0.01778986652692159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,3584,0.009252267082532246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,3072,0.017123200496037803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,3072,0.008333866794904072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,2560,0.006038400034109751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,2560,0.016952532529830932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,2560,0.007909333209196727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,2048,0.005400533477465311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,2048,0.007096533477306366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,2048,0.01625386675198873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,1536,0.004740266501903534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,4096,0.009482666850090027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,1536,0.006604800124963124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,1536,0.016008533040682473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,1024,0.004011733333269755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,768,0.0036949334045251214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,1024,0.015733333428700765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,768,0.0053727999329566956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,768,0.015466666221618653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,512,0.00342399999499321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,512,0.005173333485921224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,512,0.014828800161679586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,256,0.002994133283694585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,256,0.004747733473777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,256,0.014542933305104574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,128,0.00288426677385966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,128,0.0142794668674469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,64,0.002733866622050603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,8192,0.006674133241176605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,64,0.014354133605957031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,32,0.0027285332481066385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,65536,0.012072533369064331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,65536,0.03417706489562988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,3584,0.006596266726652781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,51200,0.01032319962978363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,3072,0.006258133550484974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,65536,0.08891733487447104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,16384,0.006932266553243001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,51200,0.02863146662712097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,16384,0.020490666230519615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,12288,0.006387199958165486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,12288,0.0199072003364563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,32,0.01442026694615682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,10240,0.006466133395830791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,10240,0.019388800859451293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,10240,0.016987733046213784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,8192,0.0063509335120519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,8192,0.018461867173512777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,16384,0.02429866592089335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,8192,0.01455466647942861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,7168,0.00625493327776591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,7168,0.019539199272791543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,7168,0.013250133395195008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,6144,0.006061866879463196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,6144,0.018626133600870766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,6144,0.012019200126330058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,5120,0.019056000312169395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,4096,0.006086400151252747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,4096,0.01813760002454122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,4096,0.009380267063776652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,1024,0.005849599838256836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,3584,0.0065290664633115125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,3584,0.01781546672185262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,3584,0.00909866690635681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,3072,0.006187733511130015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,3072,0.017560533682505288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,2560,0.006043733159701029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,2560,0.01685973405838013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,2560,0.00782293329636256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,2048,0.00543146679798762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,2048,0.01630400021870931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,5120,0.006431999802589417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,1536,0.0047189335028330484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,1536,0.015657599767049155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,5120,0.010659199953079224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,1536,0.006730666756629944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,1024,0.003991466760635376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,1024,0.006086400151252747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,1024,0.015222400426864624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,768,0.0037087999284267426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,12288,0.019681066274642944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,768,0.015042133132616677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,768,0.0053151999910672505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,512,0.003340800106525421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,512,0.005053866902987162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,256,0.0029674666623274487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,512,0.015067733327547708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,128,0.002775466690460841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,256,0.0049333333969116214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,256,0.014594133694966635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,128,0.004714666803677877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,64,0.0026943999032179515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,128,0.014453333616256715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,32,0.0026538667579491934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,64,0.014512000481287637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,32,0.014281599720319112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,65536,0.009451733032862345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,51200,0.010342400272687275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,65536,0.03206933339436849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,128,0.0046517332394917805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,16384,0.00622080018122991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,51200,0.027002666393915815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,12288,0.0061930666367212926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,10240,0.006423466900984447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,16384,0.020483199755350748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,12288,0.02030186653137207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,8192,0.0062837332487106325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,10240,0.02044373353322347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,7168,0.006209066510200501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,8192,0.018682666619618735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,7168,0.019208532571792603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,5120,0.006435200075308482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,6144,0.0186463991800944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,4096,0.005957333246866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,5120,0.019050665696461997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,3584,0.006402133405208588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,4096,0.01781760056813558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,3072,0.006080000102519989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,3584,0.017439999183019004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,2560,0.006009600063165029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,3072,0.01695573329925537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,2048,0.005398400127887726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,2560,0.016822399695714314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,1536,0.004693333307902018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,2048,0.01606186628341675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,1024,0.004043733328580856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,1536,0.015559466679890952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,768,0.0036042665441830954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,1024,0.015641599893569946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,512,0.0033215999603271483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,768,0.01515733301639557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,256,0.0029472000896930695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,512,0.014894933501879374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,16384,0.02433919906616211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,128,0.0027829334139823914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,256,0.014706133802731832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,128,0.014575999975204468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,64,0.014556800325711569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,32,0.002661333233118057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,65536,0.009065600236256917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,32,0.014206932981808982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,51200,0.008961066603660583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,65536,0.03131733338038127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,16384,0.006241066753864289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,51200,0.027412267525990804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,12288,0.006045866509278615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,16384,0.020107734203338622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,10240,0.0061930666367212926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,12288,0.019499733050664266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,8192,0.006166400015354156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,10240,0.019233065843582153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,7168,0.006108800073464712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,8192,0.018592000007629395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,6144,0.0060362666845321655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,6144,0.005927466849486033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,7168,0.018614399433135986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,5120,0.006313600142796834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,51200,0.0696821371714274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,6144,0.018488534291585288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,4096,0.0059125334024429325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,5120,0.018742400407791137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,3584,0.00621973325808843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,4096,0.01775253415107727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,3584,0.017876267433166504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,2560,0.006016000111897787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,3072,0.017387733856836955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,2048,0.005398400127887726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,2560,0.01721280018488566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,1536,0.004772266745567322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,2048,0.016217600305875143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,1024,0.004056533426046371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,1536,0.01569386621316274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,768,0.003664000084002813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,1024,0.015497600038846334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,512,0.003320533285538355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,768,0.015352533260981242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,256,0.0030986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,512,0.015005866686503092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,128,0.0028959999481836954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,256,0.014536533753077188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,64,0.002726399898529053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,128,0.014461867014567056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,32,0.0026357332865397137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,64,0.014467199643452963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,32,0.014377599954605103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,64,0.0026506667335828146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,3072,0.005859200159708659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,16384,0.7577258427937825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,3072,0.00821973333756129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,16384,0.4031253178914388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,16384,0.39060373306274415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,2048,0.007032533486684163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,51200,1.2057387034098306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,51200,1.1540351867675782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,12288,0.3079264005025228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,12288,0.2864896138509115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,12288,0.5793717066446941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,10240,0.4809578577677409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,10240,0.25853120485941566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,8192,0.21276586850484214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,10240,0.27337172826131184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,7168,0.33758719762166345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,8192,0.43260691960652664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,51200,2.3611371358235678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,7168,0.20889813105265298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,6144,0.1644810676574707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,6144,0.29283412297566735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,6144,0.15700586636861164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,5120,0.13972479502360027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,5120,0.24455040295918784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,5120,0.12773760159810382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,4096,0.1971893310546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,4096,0.11663359800974529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,4096,0.10234453678131103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,3584,0.1847914695739746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,3584,0.1042901357014974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,3584,0.09489813645680746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,3072,0.09220693111419678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,3072,0.1504554589589437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,3072,0.08094720045725504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,2560,0.12607146898905436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,2560,0.08037333488464356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,2560,0.06958826382954916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,2048,0.10341653029123943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,2048,0.06800106366475424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,2048,0.05809813340504964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,1536,0.07918186982472739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,1536,0.05542506774266561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,1024,0.06228266557057699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,1024,0.034086398283640545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,1024,0.04783573150634766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,768,0.037112534046173096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,512,0.031177600224812824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,512,0.03003093401590983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,512,0.023112533489863078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,256,0.017898666858673095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,256,0.022789333264033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,7168,0.1741066614786784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,256,0.017425066232681273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,128,0.010392533739407857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,128,0.015237333377202353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,64,0.007870933413505555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,32,0.007146666447321574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,128,0.02060906688372294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,64,0.018658133347829182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,65536,32,0.019154133399327596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,65536,768,0.04302186568578084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,768,0.030595199267069502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,8192,0.19674347241719564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,65536,1.2199551900227865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,65536,1.198902384440104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,16384,0.6992458979288737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,51200,0.9587743759155274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,65536,1536,0.046854400634765626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,16384,0.3214687983194987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,16384,0.33044586181640623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,65536,2.3735926310221354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,12288,0.4550154685974121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,51200,1.862283706665039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,10240,0.20811306635538734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,10240,0.3829941431681315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,8192,0.30360107421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,8192,0.1723061402638753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,8192,0.19537386894226075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,7168,0.26888214747111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,7168,0.15395946502685548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,6144,0.2690197308858236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,6144,0.12702399889628094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,6144,0.13295146624247234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,5120,0.19504000345865885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,5120,0.1135701338450114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,5120,0.104857603708903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,12288,0.28452800114949545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,4096,0.09557440280914306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,4096,0.08724479675292969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,4096,0.18036479949951173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,3584,0.1442581335703532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,3584,0.07636693318684896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,3072,0.11975893179575603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,10240,0.22264533042907714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,3072,0.07644906838734945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,3072,0.06659520069758097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,2560,0.10064960320790609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,2560,0.0664031982421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,2048,0.08151360352834067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,2048,0.05691946744918823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,1536,0.06303253173828124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,2048,0.04738560120264689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,1536,0.047574400901794434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,1024,0.04487680196762085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,1024,0.037780265013376876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,1024,0.028844799598058062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,768,0.03456533352533976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,768,0.0321557343006134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,512,0.025863466660181682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,512,0.026286933819452924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,51200,0.9720682779947916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,256,0.01639573375384013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,256,0.02288853327433268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,3584,0.085863463083903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,128,0.009324799974759419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,128,0.01944640080134074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,12288,0.24082132975260415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,128,0.01158186693986257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,64,0.007582933207352956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,64,0.018593066930770875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,51200,32,0.006826666494210561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,51200,32,0.018694400787353516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,7168,0.14909653663635253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,65536,0.7731583913167317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,65536,0.39361918767293297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,2560,0.05750079949696859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,65536,0.3911370595296224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,1536,0.04092053174972534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,51200,0.6036597569783528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,768,0.02471253275871277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,16384,0.18694400787353516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,512,0.02060799996058146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,51200,0.31120532353719077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,16384,0.10104640324910481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,12288,0.14158186912536622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,16384,0.12862400213877362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,12288,0.08906559944152832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,51200,0.30823678970336915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,12288,0.08825706640879313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,10240,0.07687679926554361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,10240,0.06613333225250244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,8192,0.0655839999516805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,8192,0.09652799765268961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,8192,0.06236480077107748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,7168,0.08475200335184732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,51200,256,0.013959466417630514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,7168,0.04915093183517456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,6144,0.07404373486836752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,6144,0.05373973449071249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,5120,0.0626741329828898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,5120,0.03745493491490682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,6144,0.044368000825246175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,4096,0.05111786524454752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,4096,0.04141866763432821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,3584,0.04524159828821818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,3584,0.028880000114440918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,4096,0.03163520097732544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,3584,0.03821333249409993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,3072,0.03960106770197551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,2560,0.03382506767908732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,3072,0.02595199942588806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,3072,0.035241599877675375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,2560,0.022795732816060385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,2560,0.031750400861104325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,2048,0.028097067276636762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,1536,0.022218666474024453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,2048,0.019525333245595296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,1024,0.015614933768908181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,1536,0.01611306667327881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,1536,0.024727465709050496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,1024,0.02147093415260315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,768,0.012276267011960346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,768,0.02009493311246236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,512,0.00920746624469757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,512,0.010296533505121868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,512,0.01829013427098592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,256,0.0063285330931345625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,256,0.015717333555221556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,256,0.007654400169849395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,128,0.004167466859022776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,128,0.01562666694323222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,64,0.0038592000802357995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,64,0.01591146687666575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,32,0.004099199920892716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,10240,0.12000426451365154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,32,0.01588053305943807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,7168,0.05912533203760782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,16384,5120,0.047370668252309164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,16384,2048,0.028024532397588092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,65536,0.32227840423583987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,65536,0.5724149068196615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,768,0.012018133203188579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,51200,0.4519018809000651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,16384,0.09041492938995362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,51200,0.2590005397796631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,128,0.006826666494210561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,16384,0.18064640363057455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,16384,0.09138240019480387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,51200,0.24243733088175454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,12288,0.12107840379079182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,12288,0.07287466526031494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,12288,0.06537813345591227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,10240,0.06277013222376505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,8192,0.07518719832102458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,8192,0.05429226557413737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,8192,0.0452458659807841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,7168,0.06487360000610351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,7168,0.04893120129903157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,6144,0.057316267490386964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,7168,0.040199466546376544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,6144,0.04465386470158895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,5120,0.04877013365427653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,5120,0.04016000032424927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,5120,0.030212267239888506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,4096,0.040268798669179276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,4096,0.035869868596394856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,3584,0.035410133997599284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,3584,0.03327893416086833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,3584,0.02329813241958618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,3072,0.031336534023284915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,3072,0.030598400036493938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,3072,0.020757333437601725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,2560,0.0268885334332784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,2560,0.02757013241449992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,2560,0.018347734212875368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,16384,1024,0.01285653313000997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,2048,0.022749867041905722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,2048,0.025120000044504803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,2048,0.015494400262832641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,1536,0.017864533265431724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,1536,0.02286400000254313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,1536,0.01288106640179952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,1024,0.012920533617337545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,10240,0.1033514658610026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,768,0.010217600067456563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,1024,0.020414932568868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,768,0.01039573351542155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,768,0.019181867440541588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,512,0.008157866696516674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,512,0.016526933511098227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,512,0.008272000153859456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,256,0.004730666677157084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,256,0.01586133340994517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,256,0.006786133348941803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,128,0.003917866696914038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,128,0.015175466736157736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,128,0.005860266586144766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,4096,0.025522132714589436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,64,0.0034677334129810332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,12288,32,0.0038304001092910765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,64,0.015522133310635886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,12288,32,0.01535146633783976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,65536,0.2816245396931966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,65536,0.30859092076619465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,65536,0.49230292638142903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,1024,0.010910933216412861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,51200,0.22515199979146322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,51200,0.2231680075327555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,51200,0.38819945653279625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,16384,0.07864320278167725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,10240,0.05767253239949545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,16384,0.08001813093821207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,12288,6144,0.03582613468170166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,12288,0.12098346551259358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,12288,0.058492799599965416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,12288,0.06257066726684571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,10240,0.05547840197881063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,10240,0.0904416004816691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,8192,0.06419093211491902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,7168,0.05425386826197306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,10240,0.049541334311167404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,8192,0.04734613498051961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,7168,0.043680000305175784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,7168,0.03685439825057983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,6144,0.03983680009841919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,5120,0.04054506619771321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,5120,0.027258666356404622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,4096,0.0333621342976888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,4096,0.031931734085083006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,4096,0.02355733315149943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,3584,0.029849600791931153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,3584,0.030006400744120282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,3584,0.02113706668217977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,3072,0.026521599292755126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,3072,0.027539199590682982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,3072,0.018905599912007652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,2560,0.02262399991353353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,2560,0.016476800044377647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,16384,0.13579840660095216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,2048,0.01904746691385905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,2048,0.023119999965031942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,1536,0.014788267016410828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,1536,0.021309866507848104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,1536,0.011966933806737263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,1024,0.010940800110499065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,1024,0.01927679975827535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,8192,0.04158613284428914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,1024,0.009867733716964722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,768,0.009054932991663616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,6144,0.047958401838938396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,768,0.018488534291585288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,512,0.007344000041484833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,768,0.009540266791979472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,256,0.004295466840267182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,512,0.007682133217652638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,5120,0.03580373525619507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,256,0.015609600146611533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,128,0.0037493333220481873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,128,0.005364266534646353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,128,0.015237333377202353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,64,0.0035146666069825493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,10240,32,0.003603200117746989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,64,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,32,0.015520000457763672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,65536,0.28157545725504557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,2560,0.025410133600234985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,65536,0.2215936024983724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,65536,0.4133887926737468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,51200,0.3179744084676107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,51200,0.1781941254933675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,16384,0.10330239931742351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,10240,512,0.01611199975013733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,256,0.006423466900984447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,16384,0.06614933411280313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,16384,0.06517759958902994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,12288,0.07239253520965576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,6144,0.03299306631088257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,12288,0.054680534203847254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,10240,0.060845867792765296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,12288,0.05028266509373983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,10240,0.047481600443522134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,8192,0.049600001176198324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,10240,0.042370132605234784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,8192,0.03496533234914144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,8192,0.041187198956807454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,7168,0.04388800064722697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,7168,0.03792639970779419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,6144,0.03863360087076823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,6144,0.03496640125910441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,5120,0.03282666603724162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,5120,0.031617067257563275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,4096,0.02707306742668152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,4096,0.027765333652496338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,65536,0.24695146878560387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,3584,0.027558400233586626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,10240,2048,0.014164266983668008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,3584,0.026301866769790648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,3072,0.02144320011138916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,3584,0.01881600022315979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,3072,0.02442986567815145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,2560,0.018438400824864705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,2560,0.022977066040039063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,7168,0.0306549330552419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,2048,0.015197867155075073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,2048,0.02176426649093628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,51200,0.19173653920491535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,1536,0.020840533574422202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,6144,0.027987200021743774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,1024,0.008933333555857341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,1024,0.01811520059903463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,1024,0.009720533092816671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,768,0.007597866654396057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,5120,0.023756800095240276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,4096,0.02002133329709371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,768,0.016663466890652977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,512,0.005890133480230967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,512,0.01614293356736501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,256,0.004130133241415024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,3072,0.016860800981521606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,256,0.015544533729553223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,128,0.0035242666800816857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,128,0.015299200018246969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,128,0.005769599974155426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,64,0.0033802665770053864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,2560,0.015067733327547708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,1536,0.013511466979980468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,64,0.015559466679890952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,8192,32,0.0036703998843828833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,8192,32,0.015434666474660238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,65536,0.21582293510437012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,65536,0.35892054239908855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,65536,0.2527690728505453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,2048,0.012931199868520102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,51200,0.28636372884114586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,51200,0.1706208070119222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,51200,0.1970570723215739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,16384,0.09141973654429117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,16384,0.05989120006561279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,12288,0.0642197330792745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,12288,0.050519466400146484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,10240,0.05407253503799438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,10240,0.04430186748504639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,8192,0.04411626656850179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,10240,0.04351253509521484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,8192,0.03852266470591227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,7168,0.03890133301417033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,7168,0.03528960148493449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,1536,0.011318399508794149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,6144,0.034443732102711996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,6144,0.032491733630498246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,6144,0.02901013294855754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,5120,0.02908693353335063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,5120,0.029292800029118854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,5120,0.024716800451278685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,4096,0.02416426738103231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,4096,0.025986133019129436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,512,0.007900799810886382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,768,0.008627200126647949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,3584,0.021758933862050377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,3584,0.024846933285395303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,8192,256,0.0065653334061304735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,3072,0.019257599115371705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,3072,0.023539199431737264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,3072,0.016780799627304076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,2560,0.016335999965667723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,2560,0.014839466412862143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,2048,0.013473066687583923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,2048,0.02078826626141866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,16384,0.06704853375752767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,12288,0.0511680006980896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,1536,0.01088746686776479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,2048,0.01279146671295166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,1536,0.01084266702334086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,1536,0.019413334131240845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,1024,0.008359466989835102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,2560,0.022300799687703453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,1024,0.01735573410987854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,768,0.0072543998559316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,8192,0.03679573138554891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,768,0.01572480003039042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,512,0.004901333153247834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,512,0.007355733215808869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,512,0.015714133779207863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,256,0.0037962667644023894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,256,0.015252266327540079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,128,0.003432533393303553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,7168,0.03207466602325439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,128,0.00521919975678126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,128,0.01529706617196401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,64,0.0034954667091369627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,64,0.015336533387502035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,7168,32,0.0035573333501815797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,7168,32,0.015230933825174967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,65536,0.18223360379536946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,65536,0.29531625111897786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,4096,0.020895999670028687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,51200,0.23508906364440918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,65536,0.24108586311340333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,3584,0.01876373291015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,16384,0.08383999665578207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,51200,0.14682666460673016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,16384,0.05440746545791626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,16384,0.06420373519261678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,12288,0.04646613200505574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,12288,0.06282879908879599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,10240,0.053963732719421384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,10240,0.04037333329518636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,8192,0.03868693510691325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,8192,0.035867734750111895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,8192,0.0360479990641276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,7168,0.03415786822636922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,7168,0.03284693360328674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,7168,0.03224959969520569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,6144,0.03039039969444275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,6144,0.030297599236170453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,6144,0.027962666749954224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,5120,0.026139734188715617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,5120,0.027534933884938557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,4096,0.02192853291829427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,4096,0.024897066752115886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,4096,0.02001919945081075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,3584,0.019418666760126747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,3584,0.023592533667882283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,3584,0.018266665935516357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,1024,0.008796800176302593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,3072,0.0171509325504303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,3072,0.016238933801651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,3072,0.022733867168426514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,2560,0.014510933558146158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,2560,0.021703465779622396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,768,0.008217599987983704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,2560,0.014255999525388082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,2048,0.01239466667175293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,7168,256,0.006095999975999197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,1536,0.010253866513570149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,2048,0.02034239967664083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,1536,0.010322133700052898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,1536,0.018997333447138467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,1024,0.008038400113582611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,1024,0.008310399949550629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,768,0.006785066425800323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,1024,0.01602666676044464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,512,0.004479999840259552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,768,0.008060800035794576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,768,0.015759999553362526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,256,0.0038272000849246977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,512,0.015713066856066386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,256,0.005986133217811584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,128,0.0033503999312718712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,128,0.015034666657447815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,128,0.005198933184146881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,64,0.003189333279927572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,64,0.015192533532778421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,6144,32,0.0032768001159032187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,32,0.015296000242233276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,65536,0.24105493227640787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,65536,0.15334933598836262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,51200,0.19188480377197265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,12288,0.04813226858774821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,10240,0.04148906469345093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,6144,256,0.015388799707094827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,51200,0.19004799524943033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,51200,0.12418560187021892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,16384,0.06887253125508627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,16384,0.06249493360519409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,12288,0.05414186716079712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,16384,0.05630720059076945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,12288,0.04132159948348999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,10240,0.04703573385874431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,10240,0.03634986480077108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,10240,0.04022293488184611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,8192,0.03983146746953328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,5120,0.02260479927062988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,8192,0.0322432001431783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,7168,0.029330132404963176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,7168,0.029973334074020384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,7168,0.03063360055287679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,6144,0.026050132513046265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,6144,0.027447466055552167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,2048,0.012495999534924824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,5120,0.02228906750679016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,5120,0.02553706765174866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,5120,0.022612265745798745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,6144,512,0.0071381335457166035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,4096,0.01853333314259847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,4096,0.023449599742889404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,3584,0.01644053359826406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,3584,0.022367999951044718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,3584,0.017349332571029663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,3072,0.014490666985511779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,3072,0.02132800022761027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,3072,0.01599253316720327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,2560,0.01237546702226003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,2560,0.020321067174275717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,65536,0.23582720756530762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,51200,0.18732479413350422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,2048,0.01185706655184428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,2048,0.019371734062830607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,12288,0.046779731909434005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,1536,0.00871573289235433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,1536,0.009958400328954061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,1536,0.01755946675936381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,1024,0.008459732929865519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,1024,0.01619733373324076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,768,0.005749333401521047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,768,0.0160970667997996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,512,0.004126933217048645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,512,0.015705600380897522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,8192,0.034815998872121175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,256,0.003692800054947535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,256,0.015010133385658264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,256,0.00591893345117569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,128,0.0033258666594823206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,128,0.01467626690864563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,6144,0.027167999744415285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,64,0.003123199939727783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,64,0.015011200308799743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,32,0.0032159999012947083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,5120,32,0.015215999881426492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,5120,1024,0.007045333087444305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,65536,0.1952191988627116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,4096,0.019307732582092285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,65536,0.13041600386301677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,51200,0.15533973375956217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,65536,0.21041493415832518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,51200,0.10561493237813313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,16384,0.05358719825744629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,16384,0.04597973426183065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,51200,0.1596255938212077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,12288,0.04282666842142741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,12288,0.03680320183436076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,10240,0.03680320183436076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,12288,0.03969920078913371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,2560,0.013703466455141703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,10240,0.03261546691258748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,10240,0.03517866532007853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,8192,0.03098133405049642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,8192,0.028861866394678755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,7168,0.024628265698750814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,7168,0.026778666178385417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,6144,0.021781333287556968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,6144,0.02510506709416707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,2048,0.011732266346613566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,5120,0.018667733669281004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,5120,0.023542400201161703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,768,0.007778133451938629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,4096,0.015410133202870688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,4096,0.022089600563049316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,512,0.007096533477306366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,4096,0.016820265849431356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,3584,0.013832533359527588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,3584,0.02113706668217977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,3584,0.015646933515866598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,3072,0.012219732999801636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,3072,0.020363734165827433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,2560,0.010732799768447876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,2560,0.019542400042215982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,5120,128,0.005126399795214335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,2560,0.012314666310946147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,2048,0.009242666761080424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,2048,0.018427733580271402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,1536,0.007991466422875721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,2048,0.01088213324546814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,1536,0.00923413336277008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,1536,0.01619733373324076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,1024,0.00581226646900177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,1024,0.015994667013486227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,768,0.0045952002207438145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,16384,0.05375253359476725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,768,0.015758933623631795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,8192,0.028784000873565675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,512,0.0039327998956044516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,512,0.01511679987112681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,512,0.0072512000799179075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,7168,0.02585386633872986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,256,0.0034495999415715536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,256,0.014906666676203408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,128,0.0032042667269706728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,128,0.014774399995803832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,128,0.005137066543102265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,64,0.0031008000175158186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,64,0.015092266599337259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,4096,32,0.003420799970626831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,5120,0.019387733936309815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,4096,32,0.015146666765213012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,65536,0.18111680348714193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,65536,0.12472533384958903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,3072,0.014017066359519959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,51200,0.14320640563964843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,51200,0.10213653246561687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,16384,0.0504309336344401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,6144,0.02290346622467041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,1024,0.0079434668024381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,16384,0.042843735218048094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,12288,0.03832213481267293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,12288,0.034110931555430095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,12288,0.03897813161214193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,10240,0.032858665784200033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,10240,0.030961066484451294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,10240,0.03359786669413249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,8192,0.027961599826812743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,8192,0.026894932985305785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,7168,0.021988266706466676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,7168,0.02566293279329936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,6144,0.019658666849136353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,6144,0.023933867613474526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,768,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,5120,0.016676266988118492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,5120,0.02247999906539917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,4096,256,0.006001066664854685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,4096,0.01372160017490387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,4096,0.021423999468485513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,4096,0.016145066420237223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,7168,0.025628799200057985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,3584,0.012442666292190551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,3584,0.02061226765314738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,3072,0.011171199878056844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,3072,0.019937066237131755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,3072,0.013819733262062072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,2560,0.009781333804130554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,2560,0.019127466281255088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,2560,0.012217600146929424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,2048,0.008481066425641377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,2048,0.017705599466959633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,2048,0.01062506635983785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,1536,0.007438933352629344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,1536,0.016340266664822897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,65536,0.191048526763916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,1024,0.005422933399677277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,1024,0.015804800391197204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,16384,0.05051519870758057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,768,0.004418133199214936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,51200,0.1458591938018799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,768,0.01586133340994517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,512,0.003915733347336451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,512,0.015401599804560342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,8192,0.02909333308537801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,256,0.0035594666997591654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,256,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,6144,0.023193599780400594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,128,0.0032255999743938447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,128,0.014627200365066529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,128,0.005243733525276184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,64,0.002946133414904277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,64,0.014797866344451904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3584,32,0.0030986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3584,32,0.014760532975196838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,5120,0.019181867440541588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,65536,0.1571242650349935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,65536,0.11141013304392497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,51200,0.12488959630330403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,51200,0.08904639879862467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,3584,0.015530666708946228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,16384,0.04546346664428711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,16384,0.03914133310317993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,1536,0.009030399719874065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,12288,0.033904000123341875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,1024,0.007782400151093801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,12288,0.03216746648152669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,10240,0.02927359938621521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,10240,0.028810666004816694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,768,0.00746559997399648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,8192,0.025106134017308553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,8192,0.02539199988047282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,512,0.006885333359241486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,7168,0.019735467433929444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,8192,0.03224853277206421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,7168,0.024253867069880166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3584,256,0.005799466868241628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,6144,0.017667200167973837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,6144,0.02329813241958618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,65536,0.21474773089090982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,5120,0.014932266871134438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,5120,0.02219520012537638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,51200,0.17260479927062988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,4096,0.012364799777666729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,4096,0.02059733271598816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,16384,0.05778666734695435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,3584,0.011248000462849935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,3584,0.020155733823776244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,3584,0.016377600034077962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,3072,0.010308266679445902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,3072,0.01947306593259176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,12288,0.04297066529591878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,2560,0.009212799866994222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,2560,0.01862506667772929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,10240,0.038127998510996505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,2048,0.007979733248551685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,2048,0.01665493349234263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,2048,0.010963199536005656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,1536,0.0068256000677744556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,1536,0.016999467213948568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,7168,0.028707200288772584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,1024,0.004947199920813243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,1024,0.015703466534614564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,1024,0.007877333462238312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,768,0.004375466704368591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,768,0.015434666474660238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,6144,0.02584213415781657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,512,0.00391146664818128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,512,0.01570453345775604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,256,0.0035690667728583017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,256,0.014938666423161825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,256,0.0058005332946777345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,128,0.003278933217128118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,128,0.014842666188875833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,128,0.0050016000866889955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,64,0.0029866665601730345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,64,0.014941866199175516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,3072,32,0.0030762667457262674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,3072,32,0.015012266238530478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,65536,0.13375786145528157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,5120,0.02105493346850077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,65536,0.09652799765268961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,51200,0.10904106299082439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,65536,0.18674559593200685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,512,0.007041066884994507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,4096,0.018180267016092936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,16384,0.03630613485972087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,3072,0.01479039986928304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,51200,0.07569066683451334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,51200,0.15140053431193035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,12288,0.02909119923909505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,12288,0.02986026604970296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,10240,0.025414399305979413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,10240,0.026918399333953857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,2560,0.012980266412099203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,10240,0.034382931391398114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,8192,0.02443839907646179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,7168,0.017732266585032144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,7168,0.023331199089686075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,1536,0.009239466985066731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,6144,0.015579733252525329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,3072,768,0.007696000238259633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,6144,0.02288853327433268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,16384,0.03562133312225342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,8192,0.020894932746887206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,5120,0.0132832000652949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,4096,0.011176533500353495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,5120,0.021335466702779134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,5120,0.019822933276494346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,4096,0.020147200425465903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,3584,0.010475732882817586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,4096,0.016701867183049522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,3584,0.019615999857584634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,3584,0.015233066678047181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,3072,0.009507200121879578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,3072,0.018900267283121743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,2560,0.008549333612124125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,2560,0.017407999436060587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,2048,0.007470933099587758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,2048,0.016613333423932394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,12288,0.04011733531951904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,16384,0.052184534072875974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,1536,0.006324266890684764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,1536,0.01660693287849426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,1024,0.00498879998922348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,8192,0.029895466566085816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,1024,0.01572053333123525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,1024,0.007931733131408691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,768,0.004308266441027323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,768,0.01588266690572103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,512,0.003917866696914038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,7168,0.026539733012517292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,512,0.01535040040810903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,256,0.0034613333642482757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,256,0.015209600329399109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,6144,0.024218666553497314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,3072,0.013464533289273582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,128,0.00314026673634847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,2560,0.011704533298810323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,128,0.014572800199190775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,64,0.0029535998900731405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,64,0.014702933033307395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2560,32,0.003031466652949651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2560,32,0.014706133802731832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,1536,0.008709333340326945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,65536,0.104857603708903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,65536,0.08160213629404703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,768,0.007209600011507671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,51200,0.08358826637268066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,65536,0.1947722593943278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,51200,0.06643626689910889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,16384,0.03041279911994934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,16384,0.032152533531188965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,12288,0.024499199787775674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,256,0.00572266678015391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,12288,0.027420800924301148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,10240,0.021356799205144248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,10240,0.025745066006978352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,10240,0.0315008004506429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,8192,0.01754666765530904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,8192,0.02375040054321289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,2048,0.00981119970480601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,7168,0.015953066945075988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,7168,0.022712532679239908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,6144,0.014058666427930197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,6144,0.02204479972521464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,6144,0.02149440050125122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,512,0.006660266717274983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,5120,0.013292800386746725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,5120,0.020807466904322305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,4096,0.01118293305238088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,4096,0.01959786613782247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,4096,0.015838932991027833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,3584,0.010380799571673077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,3584,0.01871573328971863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2560,128,0.0049002667268117275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,3072,0.009387733538945515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,3072,0.017870932817459106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,3072,0.012522666652997335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,2560,0.008438400427500407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,2560,0.01744106610616048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,2560,0.011457066734631855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,2048,0.007454933226108551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,2048,0.01703146696090698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,51200,0.148090664545695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,1536,0.00595413347085317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,1536,0.016484266519546507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,1536,0.008653866251309712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,1024,0.0045738667249679565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,1024,0.015587199727694193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,1024,0.007677866518497467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,768,0.004067199925581614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,768,0.01532586713631948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,768,0.0073525334397951765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,16384,0.050521600246429446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,512,0.003634133438269297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,512,0.015009066462516785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,256,0.0032266666491826378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,12288,0.03660906553268432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,7168,0.02400533358256022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,8192,0.027292799949645997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,5120,0.01803626616795858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,128,0.0030239999294281008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,64,0.0029098667204380036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,128,0.01453013320763906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,2048,32,0.002995199958483378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,64,0.01477226714293162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,32,0.014758400122324624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,3584,0.014011733730634055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,65536,0.08203306992848715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,65536,0.07124053637186686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,51200,0.06564586559931437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,51200,0.05767893393834432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,65536,0.16720107396443684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,16384,0.024540799856185912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,16384,0.027526400486628216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,16384,0.043859199682871504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,12288,0.019858133792877198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,12288,0.026603732506434125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,10240,0.01718399922053019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,12288,0.03303359945615132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,10240,0.024745599428812663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,10240,0.030110933383305866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,8192,0.014010666807492574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,8192,0.023018666108449302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,8192,0.025520000855127973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,7168,0.014284800489743552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,7168,0.0221781333287557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,6144,0.012481066584587096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,2048,256,0.014761599898338317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,2048,0.009757866462071735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,6144,0.02100693384806315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,5120,0.011672533551851908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,5120,0.020196266969045005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,4096,0.00956053336461385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,4096,0.01871573328971863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,512,0.00690773328145345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,256,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,3584,0.009041066964467366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,3584,0.01785279909769694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,2048,128,0.004956800242265066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,3072,0.008107733229796093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,3072,0.017654399077097573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,51200,0.12924266656239827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,2560,0.007456000149250031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,2560,0.01686613361040751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,7168,0.02249280015627543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,2048,0.005975466469923655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,6144,0.020338133970896403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,2048,0.016851200660069784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,1536,0.005246933301289876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,1536,0.01629866659641266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,5120,0.017066667477289833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,1024,0.004442666471004486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,1024,0.01600320041179657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,768,0.004147200038035711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,768,0.01536853313446045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,768,0.006706133484840393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,512,0.0037269333998362223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,512,0.014924800395965577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,4096,0.013944533467292786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,256,0.0032042667269706728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,1024,0.006961066524187725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,256,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,128,0.0029546665648619336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,3584,0.012914133071899415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,128,0.014747732877731323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,64,0.0028512001037597655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,64,0.014850133657455444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1536,32,0.0028970666229724885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1536,32,0.014732799927393594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,3072,0.011243733763694762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,65536,0.05535039901733398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,65536,0.05774293343226115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,65536,0.1316383997599284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,2560,0.01034453312555949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,51200,0.04854079882303874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,2048,0.009219200412432352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,51200,0.045201067129770914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,1536,0.008335999647776286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,16384,0.02413653333981832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,12288,0.013949867089589438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,16384,0.03635199864705403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,10240,0.012190933028856914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,512,0.006239999830722809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,12288,0.0231221338113149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,8192,0.010825600226720173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,10240,0.026814933617909747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,8192,0.022091732422510783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,7168,0.01011199951171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,7168,0.020992000897725425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,7168,0.019052799542744955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,6144,0.010145066181818645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,6144,0.021283199389775596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,6144,0.01729066570599874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,16384,0.017691733439763387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,51200,0.10377493699391682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,5120,0.01066986620426178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,4096,0.0092330664396286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,5120,0.013717333475748697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,5120,0.018923733631769815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,256,0.005379199981689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1536,128,0.004916266600290934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,3584,0.008434133728345235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,3072,0.00763733337322871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,10240,0.02386773427327474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,3584,0.010724266370137531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,4096,0.01801066597302755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,3072,0.01727679967880249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,3584,0.017883733908335368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,2560,0.006381866832574208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,2048,0.005579733351866404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,2560,0.00913706620534261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,2560,0.016739199558893837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,2048,0.016314666469891867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,1536,0.005010133484999338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,2048,0.00783679982026418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,1536,0.007521066566308339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,1024,0.0041514667371908825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,1536,0.01595200002193451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,768,0.0038272000849246977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,1024,0.016166399916013083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,768,0.006363733112812043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,768,0.015101866920789084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,512,0.0034645333886146545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,512,0.015020799636840821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,256,0.0032320000231266023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,256,0.014757333199183145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,256,0.005239466826121012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,3072,0.009571199615796406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,128,0.0029674666623274487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,12288,0.029051733016967774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,128,0.014658133188883463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,64,0.0028405333558718365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,1024,32,0.0028597332537174227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,64,0.014514133334159851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,1024,32,0.014531200130780539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,65536,0.044354132811228436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,65536,0.05272106726964315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,8192,0.022150399287541707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,51200,0.036266668637593584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,65536,0.1188202699025472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,16384,0.01390720009803772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,51200,0.04288746515909831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,16384,0.033215999603271484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,16384,0.023465599616368613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,12288,0.01167680025100708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,12288,0.02258239984512329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,10240,0.011147733529408772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,10240,0.02178666591644287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,4096,0.011896533767382304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,10240,0.022078933318456014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,128,0.004741333425045013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,8192,0.010007466872533162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,8192,0.019819732507069907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,7168,0.009429333607355754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,8192,0.016934400796890257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,7168,0.01911146640777588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,6144,0.009774933258692425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,7168,0.01560426652431488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,6144,0.018806399901707967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,5120,0.008810666203498841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,6144,0.013133866588274637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,4096,0.007538133362929027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,5120,0.011504000425338745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,5120,0.018900267283121743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,4096,0.00993173321088155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,4096,0.018083200852076212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,3584,0.017604267597198485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,3072,0.006512000163396199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,3072,0.017460266749064125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,1024,0.006822399795055389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,2560,0.006297599772612255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,2560,0.016846932967503867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,2560,0.008121599753697712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,2048,0.0055871998270352686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,2048,0.016622933745384216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,1024,512,0.0058442667126655575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,1536,0.004807466765244802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,1536,0.016032000382741295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,1536,0.00690773328145345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,1024,0.0041461333632469176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,1024,0.01544319987297058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,1024,0.006135466694831848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,3584,0.007032533486684163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,768,0.0037685332198937735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,768,0.015492266416549683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,512,0.0034527999659379324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,512,0.015144532918930054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,256,0.0031466667850812277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,51200,0.09269440174102783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,256,0.01486186683177948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,256,0.005013333261013031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,128,0.0029578665892283124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,12288,0.02472426692644755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,128,0.01471573313077291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,64,0.0028138667345046996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,768,32,0.0029535998900731405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,64,0.014754133423169455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,65536,0.032868266105651855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,768,32,0.014526933431625366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,65536,0.044726399580637614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,65536,0.09791786670684814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,51200,0.026609067122141523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,51200,0.03721813360850017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,51200,0.07659200032552084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,16384,0.022593067089716593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,16384,0.027108265956242876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,12288,0.010048000017801921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,12288,0.02077440023422241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,512,0.005412266651789347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,10240,0.010177066922187806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,10240,0.019588265816370645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,3584,0.009253333012262981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,128,0.004676266511281332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,8192,0.008989866574605305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,3072,0.008653866251309712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,8192,0.01458560029665629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,8192,0.018810667594273887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,7168,0.008533333738644917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,6144,0.007895466685295106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,7168,0.019169066349665323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,6144,0.01206933359305064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,5120,0.00720000018676122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,6144,0.018605866034825645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,5120,0.010664533575375874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,5120,0.018552533785502114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,4096,0.00631039987007777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,4096,0.009429333607355754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,16384,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,3584,0.006708266834417979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,3584,0.009127466877301534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,3072,0.006397866706053417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,3072,0.017051732540130614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,3072,0.008358400066693623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,2560,0.006097066899140676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,768,0.005907199780146281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,2560,0.01663040022055308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,10240,0.017449599504470826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,2048,0.00553599993387858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,2560,0.007952000200748443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,2048,0.007067733506361644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,2048,0.016229333480199178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,1536,0.004766933123270671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,1536,0.015773866573969522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,1536,0.0065994665026664736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,1024,0.004142933338880539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,768,2048,0.007255466779073079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,1024,0.015613866845766702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,1024,0.005757866799831391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,768,0.0038965334494908653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,4096,0.01805013418197632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,768,0.005480533341566721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,768,0.015204266707102457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,256,0.003070933371782303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,512,0.005305600166320801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,512,0.015034666657447815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,3584,0.01732053359349569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,128,0.002937600016593933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,256,0.014552533626556396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,128,0.014403200149536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,128,0.004686933259169261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,32,0.002726399898529053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,64,0.014416000247001648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,65536,0.02029013236363729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,512,32,0.014750933647155762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,65536,0.0363317330678304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,65536,0.0843178669611613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,51200,0.016637866695721946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,51200,0.030186667044957476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,51200,0.06744853655497232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,16384,0.00937066674232483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,16384,0.021125332514444987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,12288,0.02034453352292379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,16384,0.02373973329861959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,12288,0.008198399841785432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,10240,0.007283199826876323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,12288,0.020797866582870483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,10240,0.016812799374262492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,10240,0.019423999389012656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,8192,0.006650666892528534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,8192,0.018531199296315512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,8192,0.014418133099873862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,7168,0.006403199831644694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,7168,0.019401599963506065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,7168,0.01306773324807485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,6144,0.00702400008837382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,6144,0.018441599607467652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,6144,0.011832533280054729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,5120,0.006594133377075195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,5120,0.019102933009465535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,5120,0.010533333818117777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,4096,0.006201600035031637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,4096,0.018313600619633993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,512,0.0035445332527160645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,3584,0.006593066453933716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,3584,0.017450666427612303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,256,0.004982399940490723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,3072,0.006299733122189839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,512,64,0.002737066646416982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,2560,0.006045866509278615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,3072,0.01723626653353373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,3072,0.008402132987976074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,2048,0.005382399757703145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,2560,0.007894399762153625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,2560,0.016478932897249856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,2048,0.016291200121243795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,1536,0.004701866706212362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,1536,0.015927466750144958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,1024,0.00413226659099261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,1024,0.015656532843907674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,1024,0.005712000032265982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,768,0.0037770666182041167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,768,0.014962133765220643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,12288,0.018917334079742432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,768,0.005596800148487091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,512,0.0033728001018365227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,512,0.005184000233809153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,512,0.015079466501871744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,256,0.0047199999292691554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,128,0.0028607999285062153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,128,0.014526933431625366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,128,0.004643199841181437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,64,0.0027477333943049112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,64,0.014562132954597472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,32,0.002794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,32,0.014416000247001648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,4096,0.009469866752624512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,65536,0.01283093293507894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,65536,0.031498666604359946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,51200,0.01121493379275004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,65536,0.08592639764149984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,51200,0.02720213333765666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,16384,0.006877866884072621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,16384,0.020662399133046468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,2048,0.006958933174610138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,1536,0.006733866532643636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,16384,0.024178133408228556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,12288,0.007285333176453908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,12288,0.01936533252398173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,12288,0.019141334295272826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,10240,0.006832000116507213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,10240,0.01681706706682841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,10240,0.01919999917348226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,8192,0.006434133152167003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,8192,0.018498132626215615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,7168,0.006223999957243601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,256,256,0.003065599997838338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,7168,0.018838399648666383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,256,256,0.014987732966740927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,7168,0.013144532839457193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,6144,0.0060693333546320595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,6144,0.01181013286113739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,5120,0.006405333181222279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,6144,0.018372267484664917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,4096,0.006017066538333893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,5120,0.01876693367958069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,4096,0.009271466732025146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,4096,0.017863466342290243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,3584,0.0064746667941411335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,3584,0.017528533935546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,3584,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,3072,0.006177066763242086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,512,7168,0.013431466619173684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,3072,0.01707306702931722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,3072,0.008340266346931458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,2560,0.0060245335102081295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,2048,0.005321600039800008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,2560,0.016537599762280784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,2048,0.006941866874694824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,2048,0.01614293356736501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,1536,0.004677333434422811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,1536,0.01574399967988332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,1024,0.004036266605059306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,1024,0.015705600380897522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,1024,0.005723733206590017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,768,0.0036159999668598174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,768,0.015185067057609558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,768,0.005330133438110352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,512,0.0033962666988372804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,512,0.014872533082962037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,512,0.005029333134492239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,8192,0.014422399799029031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,256,0.0029546665648619336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,256,0.014751999576886495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,128,0.0028170667588710784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,128,0.014602667093276978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,128,0.004612266520659129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,64,0.002666666607062022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,64,0.014644267161687216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,128,32,0.0026752000053723653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,128,32,0.01455573340257009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,65536,0.011336533228556316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,65536,0.029640533526738483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,51200,0.01018346647421519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,51200,0.027324799696604413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,16384,0.006277333199977875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,16384,0.02027946710586548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,12288,0.00673280010620753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,12288,0.019154133399327596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,10240,0.00631573349237442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,51200,0.06880106925964355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,8192,0.006196266909440359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,8192,0.018500266472498576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,7168,0.006088533500830332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,7168,0.018784000476201376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,256,0.004878933231035868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,6144,0.005870933334032694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,5120,0.006345599889755249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,6144,0.01835093299547831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,4096,0.00591786652803421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,5120,0.018758400281270345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,3584,0.006322133541107178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,4096,0.017780266205469766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,3072,0.005950933198134104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,3584,0.017668267091115318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,2560,0.006054399907588959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,3072,0.01681386629740397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,2048,0.005383466680844625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,2560,0.017001599073410034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,1536,0.004648533463478088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,2048,0.015988266468048094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,1024,0.003966933240493139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,1536,0.015799466768900552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,1024,0.015655466914176942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,512,0.0033088001112143196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,5120,0.010566400488217671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,768,0.015387733777364096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,256,0.003053866575161616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,512,0.014797866344451904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,128,0.0028607999285062153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,64,0.002681600054105123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,256,0.014798933267593383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,128,0.014475733041763306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,32,0.0027189334233601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,64,0.014536533753077188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,32,0.014342400431632995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,65536,0.015641599893569946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,65536,0.0314410666624705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,51200,0.013479466239611307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,16384,0.006525866687297821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,51200,0.0282368004322052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,12288,0.006712533533573151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,16384,0.019285333156585694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,10240,0.006274133423964183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,12288,0.018796799580256145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,1536,0.006557866434256236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,8192,0.006223999957243601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,10240,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,64,10240,0.019233065843582153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,7168,0.006005333364009857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,8192,0.018343466520309448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,7168,0.01920426686604818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,6144,0.018757333358128868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,5120,0.006206933160622915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,256,3584,0.009133866429328919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,4096,0.00581226646900177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,5120,0.018822399775187175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,3584,0.0062496001521746315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,4096,0.01768746574719747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,3072,0.005927466849486033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,3584,0.01783999999364217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,2560,0.0059232001503308615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,2048,0.005321600039800008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,3072,0.017101866006851197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,2560,0.016452266772588094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,1536,0.004668800036112467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,2048,0.015961600343386333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,1024,0.00402453343073527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,1536,0.016241066654523215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,768,0.003610666592915853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,1024,0.015565866231918335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,512,0.0033013333876927694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,256,0.0029546665648619336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,512,0.014753066500027976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,128,0.0028746667007605235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,256,0.014579199751218162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,64,0.002690133452415466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,128,0.014258133371671042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,32,0.002682666728893916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,64,0.014487466216087342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,32,0.014342400431632995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,64,768,0.003638399889071783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,81,32,6144,0.00590826670328776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,16384,0.7708960215250651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,81,32,768,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,16384,0.4113834698994954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,16384,0.3759392102559408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,12288,0.5922688166300456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,81,128,2560,0.007811200122038524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,51200,1.1537994384765624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,51200,1.241601053873698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,12288,0.3132746696472168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,12288,0.28815040588378904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,10240,0.4934560139973958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,10240,0.26557226181030275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,8192,0.21696532567342125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,8192,0.19469226201375325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,10240,0.27397333780924477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,8192,0.4103263854980469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,51200,2.43701655069987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,7168,0.19300479888916017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,7168,0.1707039992014567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,7168,0.3468096097310384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,6144,0.16817173957824708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,6144,0.1508512020111084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,6144,0.298360538482666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,5120,0.12677546342213947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,4096,0.2008128007253011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,5120,0.25048534075419104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,4096,0.11828266779581706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,4096,0.10279573599497478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,3584,0.1077407995859782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,5120,0.1653546651204427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,3584,0.1788149356842041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,3072,0.15422933896382648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,2560,0.13020586967468262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,3072,0.08105279604593912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,2048,0.10569173494974773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,2560,0.08253866831461588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,2048,0.06944213708241781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,2560,0.07708693345387777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,1536,0.05879466533660889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,1024,0.056626133124033605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,1536,0.08254506587982177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,1536,0.04602133433024089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,768,0.045655465126037596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,1024,0.03435946702957153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,1024,0.049004801114400226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,768,0.030189865827560426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,512,0.032491733630498246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,512,0.03158826629320781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,256,0.0191594660282135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,256,0.01780160069465637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,128,0.011103999614715577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,128,0.020971733331680297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,128,0.015145599842071533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,64,0.008680533369382222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,64,0.019040000438690186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,32,0.007389866809050243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,32,0.018781866629918417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,3072,0.09576533635457357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,2048,0.06022506554921468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,768,0.039561601479848225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,512,0.024232532580693564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,256,0.023305600881576537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,3584,0.09217387040456136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,65536,1.2234004974365233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,51200,0.959331194559733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,51200,0.9485557556152344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,16384,0.6035167694091796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,65536,1.202947235107422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,51200,1.8478015899658202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,65536,2.3870079040527346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,16384,0.3223306655883789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,12288,0.455350399017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,12288,0.24915839831034342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,16384,0.3197663942972819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,8192,0.3040533383687337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,10240,0.23722240130106606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,10240,0.37762025197347004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,7168,0.15280319849650065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,8192,0.18371413548787435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,7168,0.27308266957600913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,6144,0.23921279907226561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,6144,0.13437867164611816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,6144,0.12768426736195881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,5120,0.19704319636027018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,5120,0.12075306574503582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,5120,0.10483307043711346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,4096,0.1577290693918864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,4096,0.10169813632965088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,4096,0.08673066298166911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,3584,0.1375264008839925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,3584,0.08621119658152263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,3072,0.12339626948038737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,3072,0.06529279947280883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,3072,0.07640213171641032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,2560,0.10017493565877278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,2560,0.06698026657104492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,2048,0.08222613334655762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,2048,0.05720959901809693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,1536,0.06302506526311238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,1536,0.05014400084813436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,12288,0.2493663946787516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,1024,0.04985280036926269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,1024,0.03799039920171102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,1024,0.028536534309387206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,10240,0.22074346542358397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,768,0.035230934619903564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,8192,0.16516159375508627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,768,0.023835732539494833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,768,0.03466133276621501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,7168,0.14636799494425456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,512,0.02693866689999898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,512,0.02831679979960124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,512,0.01930453379948934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,256,0.015403733650843302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,256,0.013977600137392679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,256,0.02143893241882324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,128,0.01016960044701894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,64,0.007114666700363159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,128,0.011577600240707397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,128,0.0196234663327535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,51200,32,0.006666666766007741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,64,0.017851734161376955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,51200,32,0.018683733542760213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,3584,0.077348264058431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,2560,0.05699413220087687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,65536,0.39901758829752604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,51200,0.5995775858561199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,2048,0.05143359899520874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,65536,0.37827733357747395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,65536,0.7634666442871094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,51200,0.31606934865315756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,16384,0.18899200757344564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,51200,1536,0.03725866476694743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,12288,0.14378347396850585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,16384,0.10156373182932536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,12288,0.07910079956054687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,16384,0.12958613236745198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,12288,0.10272640387217205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,51200,0.3057354609171549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,10240,0.07784852981567383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,10240,0.06693013509114584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,8192,0.0662986675898234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,8192,0.054731734593709315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,7168,0.0858890692392985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,7168,0.04915200074513753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,6144,0.04326826731363932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,6144,0.07501013278961181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,7168,0.06373546520868936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,5120,0.06316159963607788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,5120,0.047594666481018066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,4096,0.05134079853693644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,4096,0.04112853209177653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,3584,0.04554133415222168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,3584,0.038135464986165366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,4096,0.035214932759602864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,3072,0.03993173440297444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,3072,0.035021865367889406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,3072,0.025856000185012818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,2560,0.03400959968566895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,2560,0.03180053234100342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,2560,0.0226090669631958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,2048,0.027863466739654542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,2048,0.028233599662780762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,2048,0.019447465737660728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,1536,0.022233599424362184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,1536,0.024576000372568765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,1536,0.015897599856058757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,1024,0.015533866484959922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,1024,0.021576533714930214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,1024,0.012681600451469422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,768,0.012187733252843221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,768,0.019874133666356406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,512,0.009224533041318258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,512,0.018396800756454466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,512,0.009581866860389709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,10240,0.12067519823710124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,256,0.006275199850400289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,256,0.007496533294518788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,8192,0.09791146914164225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,128,0.004119466741879781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,256,0.015517866611480713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,64,0.003790933390458425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,128,0.0067221333583196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,128,0.015662933389345803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,32,0.00417493333419164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,6144,0.05354239940643311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,32,0.01585706671079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,64,0.015818666418393454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,5120,0.037378132343292236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,65536,0.3204383850097656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,51200,0.25781760215759275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,65536,0.3056405385335287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,51200,0.45073706309000655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,65536,0.5705216089884441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,16384,0.15378026962280272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,768,0.011367467045783997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,16384,0.09650560220082602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,16384,0.09062506357828776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,12288,0.07388052940368653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,12288,0.13817280133565266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,12288,0.06491626501083374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,10240,0.10196266969045002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,10240,0.06536639928817749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,8192,0.05620799859364828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,10240,0.05497706731160482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,8192,0.07952533562978109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,7168,0.06936000188191732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,7168,0.050955732663472496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,6144,0.06066453456878662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,6144,0.04612799882888794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,6144,0.03481386502583821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,5120,0.053003732363382974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,5120,0.04088426828384399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,5120,0.030199466149012248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,4096,0.041620266437530515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,3584,0.036813867092132566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,3584,0.033845333258310954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,3584,0.028808534145355225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,3072,0.03200746575991313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,3072,0.03136746684710185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,3072,0.02073919971783956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,2560,0.027412267525990804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,2560,0.028196267286936444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,2560,0.018066134055455527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,2048,0.02294399936993917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,2048,0.025139200687408447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,2048,0.015332266688346863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,1536,0.018037333091100057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,51200,0.2435317357381185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,1536,0.012902399897575379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,1536,0.023131734132766722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,1024,0.014334932963053385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,1024,0.010722133517265319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,768,0.010599467158317565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,1024,0.020105600357055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,512,0.008208000163237254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,768,0.00946666697661082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,768,0.01877760092417399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,7168,0.039604266484578446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,512,0.01607253352801005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,512,0.007771733403205872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,128,0.003866666555404663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,256,0.015761066476504007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,128,0.005897599955399831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,128,0.015286399920781454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,4096,0.03595199982325236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,64,0.015727999806404113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,32,0.0037674665451049804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,32,0.015779200196266174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,65536,0.27983894348144533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,65536,0.49082028071085615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,256,0.004598399996757508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,65536,0.2763242721557617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,64,0.003516799956560135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,51200,0.3870165189107259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,4096,0.02560746669769287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,8192,0.04470506509145101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,51200,0.22458987236022948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,16384,0.15398933092753092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,16384,0.0905738671620687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,16384,0.07621333599090577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,51200,0.2225269317626953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,12288,0.05959039926528931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,12288,0.06396799882253011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,12288,0.10534933408101399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,10240,0.08899733225504557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,10240,0.05118293364842733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,8192,0.04063786665598552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,8192,0.07542719841003417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,3584,0.023069866498311362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,8192,0.0481824000676473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,7168,0.055156266689300536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,7168,0.04423040151596069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,6144,0.04874879916508992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,7168,0.03606933355331421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,6144,0.04060480197270711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,5120,0.04095360040664673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,4096,0.03387306531270345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,4096,0.032219733794530234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,4096,0.02323946754137675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,3584,0.03004586696624756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,3584,0.029788800080617267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,3584,0.021245867013931274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,3072,0.02666880091031392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,3072,0.027718400955200194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,3072,0.019065600633621217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,2560,0.022692267100016275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,2560,0.016547200083732606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,2560,0.025702399015426636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,2048,0.019012266397476198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,2048,0.023578667640686037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,2048,0.014282666643460593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,1536,0.014787200093269347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,1536,0.02120959957440694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,1024,0.010726400216420491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,1024,0.019261866807937622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,256,0.0065098668138186145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,1024,0.009720533092816671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,768,0.009116799632708231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,768,0.01774186690648397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,512,0.00732479989528656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,512,0.01574613352616628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,512,0.007156266768773396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,10240,0.056304001808166505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,256,0.004277333120505015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,256,0.006016000111897787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,256,0.015641599893569946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,128,0.0037802666425704955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,128,0.015336533387502035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,64,0.003525333354870478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,5120,0.03622933228810628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,32,0.003638399889071783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,64,0.015431466698646545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,32,0.015430399775505066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,65536,0.22435305913289388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,65536,0.3902538617451986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,1536,0.011635200182596842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,51200,0.18074026107788085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,51200,0.320633602142334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,16384,0.10413333574930828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,16384,0.06818133195241292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,12288,0.07200533548990885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,12288,0.053653331597646084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,6144,0.031411200761795044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,12288,0.05109333197275797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,10240,0.06041706800460815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,10240,0.047782401243845626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,5120,0.027245867252349853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,8192,0.04952426751454671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,8192,0.04114880164464314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,7168,0.04379306634267171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,7168,0.037994666894276934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,7168,0.030320000648498536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,6144,0.03868266741434733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,6144,0.03514026800791423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,6144,0.026723200082778932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,5120,0.032961066563924155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,5120,0.03150399923324585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,768,0.008701866865158081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,4096,0.02701653242111206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,4096,0.027859199047088622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,128,0.0054848000407218935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,3584,0.024190932512283325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,3584,0.026050132513046265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,51200,0.19040106137593588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,3072,0.02138239940007528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,65536,0.25232853889465334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,3072,0.024409600098927817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,2560,0.018363734086354576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,3072,0.017196800311406454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,2560,0.022728532552719116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,16384,0.06330666542053223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,2048,0.015060266852378846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,2048,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,10240,0.043441065152486164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,1536,0.011904000242551168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,1536,0.019797333081563315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,1536,0.011191466450691223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,8192,0.03408960103988647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,1024,0.008920533458391826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,1024,0.017986132701237997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,768,0.007632000247637431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,768,0.015998933712641397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,5120,0.023341866334279378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,2560,0.014737066626548768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,512,0.006297599772612255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,512,0.016318933169047038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,256,0.0040522667268912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,4096,0.019985065857569376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,256,0.015403733650843302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,128,0.0034944000343481696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,128,0.015213867028554281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,3584,0.01895573337872823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,64,0.003339733431736628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,64,0.015459199746449789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,32,0.0036117332677046456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,32,0.01536853313446045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,65536,0.35867093404134115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,65536,0.2084671974182129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,2048,0.013006933530171714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,51200,0.28144852320353186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,51200,0.16858560244242352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,16384,0.09141546885172526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,1024,0.00918826659520467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,16384,0.05994666814804077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,51200,0.19990506172180175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,12288,0.06392639875411987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,16384,0.06541866858800252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,768,0.008151466647783916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,12288,0.04909333388010661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,10240,0.05422720114390055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,10240,0.043970131874084474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,8192,0.04385493199030559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,10240,0.04526826540629069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,8192,0.038209064801534014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,7168,0.03882453441619873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,512,0.007227733234564463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,7168,0.035811201731363936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,6144,0.03439466555913289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,6144,0.032661332686742144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,256,0.006388266881306966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,6144,0.02739306688308716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,5120,0.029230932394663494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,4096,0.024318933486938477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,5120,0.029692800839742024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,4096,0.026156800985336303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,3584,0.021693867444992066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,128,0.005756799876689911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,3584,0.024898133675257363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,3072,0.019228800137837728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,3584,0.018708266814549766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,3072,0.023663999636967976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,2560,0.016396799683570863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,3072,0.01697280009587606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,2560,0.022127999862035116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,2048,0.01344106694062551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,2048,0.020812799533208214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,2048,0.01262399951616923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,1536,0.010930132865905762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,1536,0.019637332359949747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,1536,0.010713600118954976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,1024,0.008388266960779826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,1024,0.016634666919708253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,1024,0.008494933446248371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,768,0.007208533088366191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,768,0.01535360018412272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,65536,0.24437012672424316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,512,0.004657066861788432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,512,0.015727999806404113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,12288,0.05151893297831217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,256,0.0037759999434153237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,256,0.015220266580581666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,8192,0.03707093397776286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,128,0.00344106654326121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,128,0.015169066190719605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,128,0.0052821333209673565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,7168,0.031750400861104325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,64,0.003386666625738144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,32,0.0035103999078273775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,64,0.01529813309510549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,5120,0.02376000086466471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,4096,0.020530132452646892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,32,0.015209600329399109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,65536,0.18197867075602214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,65536,0.2949546813964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,51200,0.23388694127400717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,2560,0.014837333559989929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,16384,0.08574613730112711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,51200,0.14643093744913738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,16384,0.05407679875691732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,51200,0.19200107256571453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,12288,0.0657472014427185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,12288,0.04579306840896606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,12288,0.05113600095113119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,10240,0.05583253304163614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,10240,0.04055253267288208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,8192,0.03858666817347209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,10240,0.04313173294067383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,8192,0.03528960148493449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,7168,0.034065067768096924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,7168,0.032690133651097616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,768,0.00754559983809789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,7168,0.030476800600687665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,6144,0.030342400074005127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,6144,0.03025493423144023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,5120,0.025895466407140095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,5120,0.026903466383616133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,512,0.006632533172766368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,4096,0.021262933810551964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,4096,0.024536534150441488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,4096,0.019818667570749918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,3584,0.018974934021631876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,3584,0.023161600033442177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,3584,0.01830293337504069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,3072,0.01681173245112101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,3072,0.02222506602605184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,256,0.005769599974155426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,2560,0.014493866761525472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,2560,0.02123946746190389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,2560,0.014437333742777506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,2048,0.012301866213480632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,2048,0.019997866948445638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,2048,0.012479999661445617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,1536,0.009914666414260864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,1536,0.019193599621454872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,65536,0.23563520113627115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,1024,0.008533333738644917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,1024,0.016196266810099284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,16384,0.06486186583836874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,1024,0.007869866490364075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,768,0.0068245331446329755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,768,0.01579093337059021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,512,0.004320000112056732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,512,0.015768532951672874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,256,0.0037461332976818085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,256,0.015513599912325541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,8192,0.03389333486557007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,128,0.0033312000334262846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,128,0.014957867066065469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,6144,0.026128000020980834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,64,0.003219199925661087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,64,0.015056000153223673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,32,0.0033002667129039764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,5120,0.023286400238672893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,32,0.015069866180419922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,65536,0.1550805409749349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,65536,0.24469226201375327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,51200,0.19344746271769206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,3072,0.01618133286635081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,51200,0.1251050631205241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,16384,0.06940159797668458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,51200,0.18774293263753256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,16384,0.054864001274108884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,16384,0.06019733349482218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,12288,0.05457706848780314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,12288,0.040805331865946454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,12288,0.047141333421071366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,10240,0.04746880133946736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,10240,0.036451200644175216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,8192,0.039700265725453696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,10240,0.04158506790796916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,8192,0.032181332508722946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,7168,0.029319467147191365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,7168,0.02991360028584798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,7168,0.0293503999710083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,6144,0.026060799757639568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,6144,0.027811199426651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,768,0.007346133391062419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,6144,0.02544426719347636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,5120,0.022190932432810465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,5120,0.025013333559036253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,5120,0.022817067305246987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,4096,0.018339200814565023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,512,0.00631573349237442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,4096,0.023085866371790567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,3584,0.01654293338457743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,3584,0.0221781333287557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,3584,0.01739733417828878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,3072,0.01434346636136373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,3072,0.021857066949208578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,3072,0.016317866245905557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,2560,0.012294399738311767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,2560,0.020173867543538414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,2048,0.010533333818117777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,1536,0.010137599706649781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,128,0.0050911997755368555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,2048,0.019293866554896035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,1536,0.008806399504343669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,1536,0.017614932854970296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,1024,0.006996266543865204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,1024,0.015733333428700765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,8192,0.032918399572372435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,65536,0.2309706687927246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,768,0.006072533130645752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,256,0.005649066468079885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,768,0.015847466389338174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,768,0.007361066838105519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,512,0.0040448000033696495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,512,0.006235733131567637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,512,0.01576746702194214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,256,0.003626666714747747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,256,0.015154133240381876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,256,0.005655466516812643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,128,0.0033642667035261786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,128,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,64,0.0030975999931494398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,64,0.015054933230082192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,32,0.0031530665854612983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,32,0.015077333648999533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,65536,0.19841066996256512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,65536,0.13152213096618653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,4096,0.019410133361816406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,51200,0.15751147270202637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,51200,0.10710506439208985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,1536,0.009529599547386169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,2048,0.011955199639002483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,16384,0.05367679993311564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,16384,0.045389866828918456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,51200,0.15636159578959147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,16384,0.05102399984995524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,1024,0.008032000064849854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,10240,0.03661333322525025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,10240,0.03332479993502299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,12288,0.037239468097686766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,10240,0.03563093344370524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,8192,0.0287445326646169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,8192,0.027771733204523724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,7168,0.027876265843709308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,7168,0.026410667101542155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,7168,0.02485333283742269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,6144,0.0249674657980601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,6144,0.025166932741800947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,6144,0.0216757337252299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,5120,0.018739199638366698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,5120,0.023425066471099855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,5120,0.019847466548283895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,4096,0.01528320014476776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,4096,0.021769599119822184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,3584,0.013771733641624451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,3584,0.021093332767486574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,3584,0.01567039986451467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,3072,0.012277332941691081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,3072,0.02034026583035787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,128,0.005074133475621542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,3072,0.013808000087738036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,2560,0.010665599505106609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,2560,0.019348265727361043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,2048,0.009076266487439474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,2048,0.01845546762148539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,12288,0.04256639877955119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,2048,0.010940800110499065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,1536,0.007776000102361043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,1536,0.008887466788291932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,1024,0.006168533364931742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,8192,0.03076266646385193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,1024,0.015638400117556253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,1024,0.007584000130494435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,768,0.004274133344491323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,768,0.015403733650843302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,512,0.0038453333079814913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,512,0.015289599696795145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,65536,0.20595733324686685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,2560,0.014006400108337402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,256,0.003509333233038584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,256,0.005629866818586985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,256,0.014942933122316995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,128,0.0030879999200503034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,128,0.014893866578737893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,64,0.003020799905061722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,64,0.015201066931088766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,32,0.003307733436425527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,32,0.015315199891726175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,1536,0.01626240015029907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,65536,0.18402454058329265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,65536,0.1260703961054484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,12288,0.04109013477961222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,65536,0.1876245339711507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,51200,0.1465354601542155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,51200,0.10022933483123779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,16384,0.05036906798680624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,51200,0.1485930601755778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,16384,0.04261120160420735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,16384,0.04967999855677287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,12288,0.03793600002924601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,12288,0.03662826617558797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,12288,0.03983146746953328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,10240,0.03279680013656616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,10240,0.03067413369814555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,8192,0.02765546639760335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,10240,0.03480960130691528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,4096,0.01660480002562205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,7168,0.025081600745519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,8192,0.027540266513824463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,7168,0.025378133853276568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,7168,0.02474986712137858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,6144,0.022477867205937703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,6144,0.02400319973627726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,6144,0.021893332401911415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,5120,0.01649066706498464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,5120,0.022613332668940226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,4096,0.013981866836547851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,4096,0.0213536004225413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,2560,0.01244586706161499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,4096,0.01652479966481527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,3584,0.012350933750470479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,3584,0.02063360015551249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,3584,0.015377066532770791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,3072,0.011442133784294128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,3072,0.019935999313990274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,2560,0.009860266248385112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,2560,0.019040000438690186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,2560,0.01221226652463277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,768,0.007090133428573608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,2048,0.00865600009759267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,2048,0.016849066813786825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,1536,0.0073738664388656614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,512,0.006325333317120869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,1536,0.01655359963575999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,1024,0.005295999844868978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,1024,0.01572160025437673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,8192,0.02693120042483012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,1024,0.007310933371384938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,768,0.00421013335386912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,768,0.015689599514007568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,768,0.006865066786607106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,512,0.003818666686614355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,512,0.015155200163523355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,256,0.003370666752258936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,256,0.014621866742769876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,128,0.005173333485921224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,256,0.005492266515890757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,128,0.003160533308982849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,128,0.005067733426888784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,128,0.014539733529090881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,64,0.002946133414904277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,32,0.003101866692304611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,64,0.014682666460673014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,32,0.014869333306948344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,65536,0.15764373143513996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,65536,0.10999680360158284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,51200,0.12441066900889079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,65536,0.22117226918538413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,51200,0.08855679829915365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,16384,0.04533226490020752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,16384,0.03851840098698934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,512,0.006179200112819671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,12288,0.03327573339144389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,12288,0.033297065893809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,10240,0.028769065936406452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,5120,0.019208532571792603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,10240,0.02980479995409648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,10240,0.03872106472651164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,8192,0.024371200799942018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,8192,0.02699413299560547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,7168,0.021951999266942343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,3072,0.013708800077438354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,7168,0.0242741326491038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,7168,0.027574400107065838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,6144,0.0232149342695872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,2048,0.010706133643786113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,1536,0.00890773336092631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,5120,0.01683626572291056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,5120,0.0221781333287557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,5120,0.02018773357073466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,4096,0.014500266313552857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,4096,0.020806399981180827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,4096,0.01800959904988607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,3584,0.011288533608118694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,3584,0.020373332500457763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,3072,0.010243200262387593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,3072,0.01958079934120178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,6144,0.019655466079711914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,2560,0.009218133489290873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,2560,0.01840959986050924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,2560,0.01318826675415039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,2048,0.008025600016117096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,2048,0.017643733819325765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,51200,0.17836693127950032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,16384,0.05470293362935384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,1536,0.007435733576615651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,1536,0.01632960041364034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,1536,0.00897173285484314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,1024,0.004644266764322917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,1024,0.015633066495259605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,768,0.0043029333154360454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,12288,0.04362346728642781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,768,0.015296000242233276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,768,0.007004799942175548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,512,0.0037845333417256674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,512,0.01527253290017446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,256,0.0033834666013717652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,256,0.015072000026702882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,8192,0.030820266405741377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,6144,0.023405865828196207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,128,0.003156266609827677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,128,0.014745600024859109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,128,0.005156266689300537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,64,0.002962133288383484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,64,0.014850133657455444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,32,0.0030634666482607523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,32,0.01477120021979014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,65536,0.13546773592631023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,3584,0.01649173299471537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,3072,0.014751999576886495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,65536,0.18539199829101563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,51200,0.10823787053426107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,51200,0.07549760341644288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,16384,0.03599040110905965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,16384,0.035548798243204754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,16384,0.05188479820887247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,12288,0.028777599334716797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,12288,0.0301962673664093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,2048,0.010915199915568035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,10240,0.02521599928538005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,10240,0.026667733987172444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,65536,0.0966048002243042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,1024,0.007423999905586243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,8192,0.020950400829315187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,8192,0.02866026759147644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,7168,0.018887466192245482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,512,0.0060586666067441305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,7168,0.02345386743545532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,6144,0.016782933473587038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,6144,0.022309333086013794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,256,0.005401599903901418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,5120,0.013187199831008911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,5120,0.021171200275421142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,5120,0.01965013345082601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,4096,0.011106133460998535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,4096,0.020139733950297035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,3584,0.010416000088055929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,3584,0.019526400168736777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,8192,0.02467306653658549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,3072,0.00944533348083496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,3072,0.01894186735153198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,3072,0.013645866513252258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,2560,0.008447999755541485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,2560,0.01744106610616048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,51200,0.15210879643758138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,2048,0.007968000074227651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,2048,0.016509866714477538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,12288,0.040756265322367355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,1536,0.006403199831644694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,1536,0.01615466674168905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,1536,0.008462933699289958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,1024,0.004524800181388855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,10240,0.03563093344370524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,1024,0.015920000274976094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,768,0.004121600091457367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,768,0.01578986644744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,7168,0.026537599166234334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,512,0.003714133302370707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,512,0.014961066842079162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,6144,0.02257173260052999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,256,0.0033632000287373864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,256,0.014920533696810404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,256,0.0053951998551686605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,128,0.0031338666876157125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,4096,0.016794667641321818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,128,0.014751999576886495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,64,0.003011200080315272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,64,0.01480959951877594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,32,0.003031466652949651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,3584,0.015292800466219583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,32,0.014682666460673014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,65536,0.1059114694595337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,65536,0.08038933277130127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,51200,0.084224001566569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,51200,0.06650453408559164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,2560,0.011745066444079081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,16384,0.030078933636347456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,16384,0.031923200686772665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,2048,0.009746133287747701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,12288,0.024435200293858848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,12288,0.027479465802510577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,12288,0.039033599694569904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,10240,0.02123946746190389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,10240,0.02539413372675578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,1024,0.007231999933719635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,10240,0.032579199473063154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,8192,0.017383466164271034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,8192,0.024755199750264488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,7168,0.015966932972272238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,768,0.006690133114655812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,7168,0.02318613330523173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,6144,0.014307199915250143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,6144,0.02225066622098287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,6144,0.020348799228668214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,5120,0.013617066542307535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,5120,0.020755199591318767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,5120,0.017375999689102174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,4096,0.01202880044778188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,512,0.00609493354956309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,4096,0.01970240076382955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,128,0.005065600077311197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,3584,0.011286399761835734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,3584,0.018026665846506754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,3584,0.013962666193644205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,3072,0.010263466835021972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,3072,0.017384533087412515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,3072,0.012584533294041952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,2560,0.007794133325417836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,65536,0.18559573491414388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,2560,0.01678826610247294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,2048,0.007021866738796234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,2048,0.016545066237449647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,51200,0.14565547307332355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,1536,0.005348266661167144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,1536,0.01625920037428538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,1536,0.008371200164159138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,1024,0.004345599810282389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,1024,0.015602133671442666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,1024,0.007144533097743988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,768,0.003977599988381068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,768,0.015479466319084168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,16384,0.046284798781077066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,512,0.003638399889071783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,512,0.015554133057594299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,512,0.005904000004132589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,256,0.00325546662012736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,256,0.014941866199175516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,8192,0.02587733268737793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,128,0.003070933371782303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,128,0.014590932925542196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,7168,0.023630932966868082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,64,0.002902399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,64,0.014588800072669984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,32,0.0029546665648619336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,32,0.014696533481280008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,65536,0.08300053278605143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,65536,0.07082560062408447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,4096,0.015441067020098367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,51200,0.06593386729558309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,65536,0.16120533943176268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,51200,0.05735253492991129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,16384,0.02437973419825236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,16384,0.027640533447265626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,16384,0.043747198581695554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,12288,0.019385600090026857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,12288,0.02413653333981832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,12288,0.03375680049260457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,2560,0.01120853324731191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,10240,0.030123732487360638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,10240,0.02318399945894877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,8192,0.013927466670672097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,8192,0.02172693411509196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,8192,0.024649600187937416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,7168,0.012807466586430869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,7168,0.023627734184265135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,6144,0.011560533444086711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,7168,0.02254826625188192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,6144,0.02245546579360962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,6144,0.019310933351516724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,5120,0.01048959990342458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,5120,0.020497065782546998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,5120,0.01688533425331116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,4096,0.009611733754475911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,4096,0.018573866287867228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,3584,0.010753066341082255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,3584,0.017624533176422118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,768,0.006798933446407318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,3072,0.009609599908192951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,3072,0.017416532834370932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,3072,0.012013866504033407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,2560,0.0075103998184204105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,2560,0.01685439944267273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,256,0.0053941334287325535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,2560,0.010272000233332317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,2048,0.005671466886997223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,2048,0.016437333822250367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,1536,0.004955733319123586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,10240,0.016733866930007935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,1536,0.016176000237464905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,1024,0.004260266820589701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,1536,0.007921066880226136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,2048,0.009668266773223877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,768,0.003962666789690653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,768,0.015707733233769734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,128,0.004990933338801066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,768,0.006311466793219249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,512,0.003656533360481262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,512,0.015396266182263692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,512,0.005580799778302511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,256,0.0032277333239714304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,256,0.005179733534653981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,256,0.01492800017197927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,128,0.0030613332986831666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,128,0.014643200238545737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,128,0.00492799977461497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,64,0.002825599908828735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,32,0.0029109333952267963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,64,0.014724266529083253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,32,0.014787200093269347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,65536,0.054712533950805664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,65536,0.05782399972279867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,65536,0.1315658648808797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,51200,0.1290549357732137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,51200,0.04429759979248047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,51200,0.04784853458404541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,16384,0.01832746664683024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,51200,0.1030890703201294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,16384,0.0240341325600942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,16384,0.03688533306121826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,12288,0.014058666427930197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,12288,0.025961599747339886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,12288,0.029121067126591998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,10240,0.01232319970925649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,10240,0.02421226700146993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,8192,0.010770133137702942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,8192,0.022428800662358604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,8192,0.021281067530314127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,7168,0.010160000125567118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,7168,0.021513599157333373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,4096,0.0142794668674469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,6144,0.012107732892036437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,3584,0.01295360028743744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,6144,0.020758400360743202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,5120,0.01064639985561371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,5120,0.020754132668177286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,5120,0.013802666465441385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,1024,0.01590079963207245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,4096,0.009237333138783773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,2048,0.008826667070388794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,4096,0.018436266978581747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,3584,0.008588799834251403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,3584,0.017654399077097573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,3584,0.010920533537864685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,10240,0.025617067019144697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,3072,0.007742933432261149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,3072,0.009916800260543823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,3072,0.017538134256998697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,2560,0.006958933174610138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,2560,0.009062400460243225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,1024,0.00655680000782013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,2560,0.016871466239293417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,2048,0.005638400216897329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,2048,0.016313599546750386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,1536,0.015954132874806723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,1536,0.006908800204594929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,1024,0.004343466460704803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,1024,0.006049066781997681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,1024,0.01575146714846293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,768,0.004005333284536997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,768,0.015389866630236306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,768,0.005871999760468801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,512,0.003589333345492681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,512,0.005358933409055074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,256,0.0031839999059836066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,256,0.014732799927393594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,256,0.005008000135421753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,128,0.002906666696071625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,128,0.014647466937700906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,64,0.002855466554562251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,64,0.014599466323852539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,32,0.0028618666032950084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,32,0.014780799547831217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,4096,0.011815466483434041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,65536,0.04407466650009155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,65536,0.05178240140279135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,65536,0.11636373202006023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,51200,0.03601386547088623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,1536,0.005098666747411092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,51200,0.04332586526870728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,51200,0.08917439778645833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,16384,0.014123732844988504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,16384,0.024477867285410564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,12288,0.011788800358772278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,16384,0.030824534098307294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,10240,0.010734933614730834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,12288,0.024625066916147867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,12288,0.022900267442067464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,8192,0.011334400375684102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,10240,0.02178879976272583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,10240,0.021364265680313112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,7168,0.010616532961527507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,8192,0.016613333423932394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,512,0.015066666404406228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,8192,0.020577067136764528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,6144,0.009845333298047383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,7168,0.019818667570749918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,6144,0.019512534141540527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,5120,0.008901333808898926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,7168,0.018747733036677042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,5120,0.01877440015474955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,4096,0.007659733295440674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,4096,0.018228266636530557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,4096,0.009853866696357728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,3584,0.007131733496983846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,3584,0.017844265699386595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,3584,0.00938986639181773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,3072,0.006737066805362702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,3072,0.01763413349787394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,6144,0.016214399536450704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,2560,0.006237866481145223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,2560,0.01686186591784159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,2560,0.008078933258851369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,2048,0.005566933254400889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,2048,0.01626240015029907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,2048,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,1536,0.004759466648101807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,1536,0.01599360009034475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,1536,0.00662720004717509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,1024,0.00413226659099261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,1024,0.015478400389353433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,1024,0.005747200051943461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,2048,0.0076799998680750535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,768,0.0037493333220481873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,768,0.005571199953556061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,768,0.015378133455912272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,512,0.0034944000343481696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,6144,0.01309333344300588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,512,0.005166933437188466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,512,0.01514346698919932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,256,0.01474346617857615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,256,0.0048991998036702475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,128,0.002898133297761281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,128,0.004693333307902018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,64,0.0028266665836175283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,64,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,32,0.002880000074704488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,32,0.0144405335187912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,65536,0.03252159953117371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,128,0.004785066843032837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,65536,0.04448426564534505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,51200,0.02696320017178853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,51200,0.0368938684463501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,51200,0.07585706710815429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,16384,0.01106666624546051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,16384,0.022361600399017335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,16384,0.026768000920613606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,12288,0.00993280013402303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,12288,0.02184106707572937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,12288,0.02084160049756368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,10240,0.010311466455459595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,256,0.003126399964094162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,10240,0.02134079933166504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,10240,0.017309866348902383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,8192,0.009139200051625569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,128,0.014533332983652749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,8192,0.014485333363215128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,8192,0.01978773276011149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,7168,0.00909866690635681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,7168,0.014812800288200378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,7168,0.013294933239618936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,7168,0.019186133146286012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,6144,0.008961066603660583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,6144,0.011821867028872172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,5120,0.008326399823029835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,6144,0.018691200017929076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,4096,0.007266133526961009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,5120,0.010637866457303365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,5120,0.011846400300661723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,5120,0.01889280080795288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,4096,0.018268799781799315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,4096,0.009452799956003826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,3584,0.006776533524195353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,3584,0.009244799613952637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,3584,0.018078933159510292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,3072,0.006431999802589417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,2560,0.006093866626421611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,3072,0.00829013337691625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,2560,0.007880533238252004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,2560,0.016676266988118492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,1536,0.004816000163555145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,2048,0.007085866729418437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,3072,0.00862506628036499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,2048,0.016306133071581522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,1024,0.004173866907755534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,1536,0.01590720017751058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,768,0.0038293334345022834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,1024,0.015404799580574035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,768,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,768,0.005357866485913595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,512,0.005182933310667674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,512,0.014936533570289613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,256,0.0031776001056035364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,256,0.004862933357556661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,256,0.014544000228246054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,128,0.002921599894762039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,128,0.004758400221665701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,64,0.002746666719516118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,64,0.014665599664052328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,32,0.0028042666614055633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,32,0.014572800199190775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,65536,0.020051199197769164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,65536,0.0361792008082072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,65536,0.08536960283915201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,51200,0.01809813380241394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,51200,0.030745599667231244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,51200,0.0670144001642863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,3072,0.017182934284210204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,16384,0.009431466460227966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,2048,0.005500799914201101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,16384,0.020536533991495767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,16384,0.0237610658009847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,1536,0.006620799998442332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,12288,0.019042134284973145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,12288,0.0195360004901886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,512,0.00345920001467069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,10240,0.00824426660935084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,8192,0.007187200089295705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,10240,0.019486933946609497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,8192,0.01872426668802897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,8192,0.014345600207646688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,7168,0.006622933348019918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,65536,0.09664959907531738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,7168,0.019074134031931558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,128,0.014492799838383993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,7168,0.013149866461753845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,6144,0.006259199976921081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,5120,0.006604800124963124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,6144,0.01169706682364146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,5120,0.010517332951227825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,4096,0.006131199995676676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,5120,0.018780799706776936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,3584,0.00652159998814265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,4096,0.00960106650988261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,4096,0.01801813244819641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,3072,0.006198399762312571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,3584,0.00909440020720164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,3584,0.01766506632169088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,2560,0.0060586666067441305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,3072,0.008246399958928426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,3072,0.017139200369517008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,2048,0.005388799806435903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,2560,0.007914666831493378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,2560,0.016739199558893837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,1536,0.004727466901143392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,2048,0.016386133432388306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,1536,0.01579839984575907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,1024,0.004091733445723852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,1024,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,1024,0.005674666663010915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,768,0.0036415999134381616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,768,0.015346133708953857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,768,0.0053162669142087305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,512,0.0033781332274278007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,12288,0.009075199564297993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,512,0.015065600474675497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,256,0.0030389333764712016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,512,0.005137066543102265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,256,0.00480320006608963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,256,0.01477013329664866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,128,0.0028021333118279776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,128,0.01460906664530436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,128,0.004604800045490265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,64,0.002678400029738744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,32,0.0026890667776266735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,64,0.014430933197339377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,1024,0.005747200051943461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,32,0.014578133821487427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,65536,0.012516267100969949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,51200,0.01087679962317149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,6144,0.018551466862360637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,65536,0.03147520025571187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,65536,0.08619733651479086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,51200,0.02757226626078288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,51200,0.06927893161773682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,16384,0.02019946575164795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,12288,0.00697386662165324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,16384,0.0240447998046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,10240,0.006493866443634033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,12288,0.019241599241892497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,10240,0.01684160033861796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,10240,0.01914986570676168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,8192,0.006386133531729381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,8192,0.018655999501546224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,8192,0.014450132846832275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,1536,0.006600533425807953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,7168,0.006263466676076253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,7168,0.01900586684544881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,7168,0.013201066851615905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,6144,0.005995733539263407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,5120,0.006489600241184235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,6144,0.011785599589347839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,6144,0.018683733542760213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,4096,0.0060928001999855045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,5120,0.010598400235176086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,5120,0.01877760092417399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,4096,0.009271466732025146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,4096,0.018278400103251137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,3584,0.0065610667069753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,3584,0.009099733829498292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,3584,0.017997866868972777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,3072,0.006082133452097575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,3072,0.008152533570925396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,2560,0.006041599810123444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,3072,0.017068799336751303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,2048,0.005410133302211762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,2560,0.007712000111738841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,2560,0.01690666675567627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,1536,0.005098666747411092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,2048,0.006965333223342895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,2048,0.016286933422088624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,1536,0.006687999765078227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,1536,0.015987199544906617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,1024,0.0040949332217375435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,1024,0.01527679959932963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,1024,0.005706666906674703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,768,0.0036490666369597114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,512,0.0033845332761605583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,768,0.015341867009798685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,512,0.015140266219774882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,512,0.004997333387533823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,256,0.0030048000315825146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,256,0.01495039959748586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,256,0.004948266843954722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,128,0.0028607999285062153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,16384,0.007314133147398631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,128,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,128,0.004613333443800608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,64,0.002716800073782603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,32,0.002625066787004471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,64,0.014499200383822122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,32,0.014412800470987955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,12288,0.020003199577331543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,65536,0.00970240036646525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,51200,0.00881706674893697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,65536,0.029662932952245074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,16384,0.006185600161552429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,51200,0.02540053327878316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,12288,0.006141866743564606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,10240,0.006301866471767425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,16384,0.020730666319529214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,12288,0.019323732455571493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,10240,0.01885973413785299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,7168,0.006108800073464712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,8192,0.00626453310251236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,8192,0.018578133980433145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,6144,0.00591786652803421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,5120,0.006377600133419037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,6144,0.01827733318010966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,4096,0.005950933198134104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,5120,0.018580265839894614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,4096,0.017972266674041747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,3584,0.006337066491444905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,3072,0.006098133325576782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,3584,0.017205333709716795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,2560,0.005975466469923655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,3072,0.017619200547536216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,2048,0.0053375999132792154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,2560,0.016177067160606386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,1536,0.004691199958324432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,2048,0.016060800353686015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,1024,0.004066133250792822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,1536,0.015581867098808289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,768,0.003714133302370707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,1024,0.01527253290017446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,768,0.015258666872978211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,256,0.0029205332199732465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,512,0.01474666694800059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,128,0.00275093341867129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,256,0.014443733294804893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,64,0.0027104000250498454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,128,0.014346667130788169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,32,0.002631466587384542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,64,0.014272000392278036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,65536,0.011403733491897583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,32,0.01415786643822988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,65536,0.028645332654317218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,51200,0.009682133793830872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,16384,0.006343466540177663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,51200,0.027323732773462932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,16384,0.020134399334589638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,12288,0.006110933423042297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,10240,0.006296533346176148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,12288,0.01888213356335958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,8192,0.006159999966621399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,10240,0.019402666886647543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,7168,0.005975466469923655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,8192,0.018547199169794717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,6144,0.005930666625499725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,7168,0.018689066171646118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,5120,0.00621973325808843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,6144,0.018247467279434205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,7168,0.01862293283144633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,4096,0.005851733187834421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,5120,0.018528000513712565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,3584,0.0063381334145863845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,4096,0.01775040030479431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,3584,0.017321600516637167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,3072,0.005993600189685822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,2560,0.0060127998391787205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,3072,0.016921599706014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,2048,0.005352533360322317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,2560,0.01641813317934672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,1536,0.004658133288224538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,2048,0.015962666273117064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,1024,0.003940266619126002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,1536,0.01570026675860087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,768,0.003585066646337509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,512,0.0033258666594823206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,512,0.0033514666060606635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,768,0.014973866939544677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,256,0.0029546665648619336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,512,0.015185067057609558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,128,0.0028064000109831494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,10240,0.01679146687189738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,256,0.014437333742777506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,64,0.0026464000344276427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,128,0.014409599701563516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,32,0.0026890667776266735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,64,0.014200533429781595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,32,0.014505599935849508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,2048,0.007011199990908305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,1024,0.015470932920773825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,16384,0.7688063939412435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,16384,0.40544961293538406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,16384,0.3858549435933431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,12288,0.3127989451090495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,12288,0.5926197052001954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,51200,1.1639146169026693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,51200,1.235771687825521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,768,0.0054175997773806255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,12288,0.29321066538492835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,10240,0.26797440846761067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,10240,0.5023285230000814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,8192,0.2172821362813314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,8192,0.20947093963623048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,10240,0.25547839800516764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,51200,2.4349822998046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,7168,0.34595947265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,7168,0.19157333374023439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,7168,0.18303573926289876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,6144,0.29734719594319664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,6144,0.16698880195617677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,6144,0.15777173042297363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,5120,0.14263466199239094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,5120,0.13143359820048015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,4096,0.10649279753367107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,4096,0.20141973495483398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,4096,0.1192522684733073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,5120,0.29409173329671223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,3584,0.1782485326131185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,3584,0.09433920383453369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,3072,0.1532629330952962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,3584,0.10555840333302816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,3072,0.081987198193868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,3072,0.09433386325836182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,2560,0.08196799755096436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,2560,0.15000319480895996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,2560,0.0712170680363973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,2048,0.1035648028055827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,1536,0.08094613552093506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,2048,0.05849066575368246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,1024,0.055275734265645346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,1536,0.062135465939839686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,768,0.044760533173878986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,1024,0.04418026606241862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,1024,0.034024532636006674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,768,0.03834346532821655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,512,0.03174506624539693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,768,0.030177066723505657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,256,0.0183786670366923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,512,0.03068586587905884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,512,0.022663466135660806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,256,0.017035732666651406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,256,0.022860799233118692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,64,0.0081194669008255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,128,0.02035413384437561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,64,0.01855573256810506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,32,0.006562133133411407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,32,0.018802134195963542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,65536,2048,0.06838186581929526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,1536,0.0499722679456075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,128,0.010733866691589355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,65536,128,0.01529706617196401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,65536,8192,0.40285011927286785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,16384,0.3343157450358073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,16384,0.6013589223225911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,65536,1.2786837259928385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,65536,1.2282549540201821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,16384,0.3339263916015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,12288,0.2994613329569499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,51200,1.0170687993367513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,51200,0.9865407943725586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,65536,2.3737078348795575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,10240,0.37718400955200193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,10240,0.21833813985188805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,10240,0.2309951941172282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,51200,1.9301130930582684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,7168,0.2666218598683675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,7168,0.1499850591023763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,8192,0.3240394592285156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,6144,0.13285760084788006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,6144,0.23903466860453287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,12288,0.4472949345906575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,5120,0.19293227195739746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,5120,0.11804160277048747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,4096,0.15701546669006347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,4096,0.09456533590952555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,5120,0.1181109348932902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,3584,0.1357749303181966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,4096,0.0946943998336792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,3584,0.0846442699432373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,3072,0.12222399711608886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,8192,0.19319146474202473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,3072,0.07511359850565592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,3072,0.06863893667856852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,2560,0.10153813362121582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,2560,0.06561813354492188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,2560,0.062149333953857425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,2048,0.08153599898020426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,2048,0.06161386569341024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,1536,0.06202666759490967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,1536,0.046930134296417236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,12288,0.26804800033569337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,1536,0.037853864828745525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,1024,0.03711466789245606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,1024,0.049236265818277995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,768,0.03827413320541382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,768,0.03184319933255513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,8192,0.1817237377166748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,512,0.028018132845560713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,512,0.025844266017278034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,512,0.020386133591334024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,256,0.01349546710650126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,7168,0.1649504025777181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,256,0.02095680038134257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,128,0.00925546685854594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,6144,0.1410304069519043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,2048,0.05250666538874308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,128,0.01770026683807373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,64,0.006587733328342438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,51200,32,0.006137600044409434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,64,0.017960532506306966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,51200,32,0.018105600277582803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,3584,0.08061227003733316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,65536,0.3937525431315104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,65536,0.7649482727050781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,256,0.014086400469144186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,65536,0.3882815996805827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,16384,0.18787946701049804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,51200,0.3122314771016439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,16384,0.11230186621348064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,51200,0.5965834935506185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,16384,0.10415253639221192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,12288,0.14330347379048664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,12288,0.09019412994384765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,12288,0.0887168010075887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,10240,0.11983466943105062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,1024,0.0321557343006134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,10240,0.06911893685658774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,10240,0.07667840321858724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,8192,0.057214931646982825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,8192,0.06529920101165772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,7168,0.08533759911855063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,8192,0.09682880242665609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,7168,0.058723199367523196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,768,0.025915733973185223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,7168,0.050153601169586184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,5120,0.06254613399505615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,6144,0.0530730684598287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,6144,0.04385066827138265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,5120,0.046876800060272214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,5120,0.03766080141067505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,4096,0.04096746842066447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,4096,0.0318175991376241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,3584,0.045254401365915936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,3584,0.03798720041910807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,3584,0.028889600435892743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,3072,0.03949546813964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,3072,0.03458346525828044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,3072,0.02566293279329936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,2560,0.033752532800038655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,2560,0.03134613235791524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,2560,0.022555732727050783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,2048,0.027992532650629683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,2048,0.019371734062830607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,1536,0.021943465868632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,1024,0.01541973352432251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,1536,0.015657599767049155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,1536,0.024405332406361897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,768,0.012098133563995361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,1024,0.012837333480517068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,1024,0.021116799116134642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,512,0.00902400016784668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,768,0.012103466192881267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,768,0.019869865973790486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,256,0.006367999811967213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,512,0.009960533181826273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,51200,128,0.010668800274531046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,512,0.018081067005793254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,51200,0.3248490651448568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,128,0.003931733220815659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,256,0.007537066439787547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,256,0.015948800245920818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,128,0.015460266669591268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,64,0.0036789332826932275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,32,0.003902933249870936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,16384,128,0.006597333153088887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,32,0.01588586668173472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,6144,0.07432746887207031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,4096,0.05113600095113119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,65536,0.31611626942952475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,65536,0.3265088081359863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,65536,0.5677087783813477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,16384,2048,0.027722666660944622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,16384,0.15515200297037762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,16384,0.10555306275685628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,16384,0.08650453090667724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,51200,0.2528170744578043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,12288,0.12003200054168701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,51200,0.25283734003702796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,51200,0.44760319391886394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,12288,0.06575466791788737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,12288,0.07467733224232992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,10240,0.06431253353754679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,8192,0.05524906714757284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,8192,0.047414398193359374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,10240,0.057581865787506105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,16384,64,0.015404799580574035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,8192,0.0905184030532837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,7168,0.06859412988026937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,7168,0.04099306662877401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,6144,0.060031998157501223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,6144,0.036294400691986084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,5120,0.04014933506647746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,5120,0.03158400058746338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,4096,0.041330134868621825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,4096,0.03531519969304402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,4096,0.0265066663424174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,3584,0.03674346605936686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,3584,0.032865067323048905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,3584,0.02411093314488729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,3072,0.02996266682942708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,3072,0.021590399742126464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,2560,0.027166932821273804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,2560,0.027628799279530842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,2560,0.019151999553044637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,2048,0.022616533438364665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,2048,0.025102933247884113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,1536,0.017754666010538735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,1536,0.02287786602973938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,1536,0.01323946714401245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,1024,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,1024,0.020143999656041463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,1024,0.011372799674669903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,768,0.010518399874369304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,768,0.01883413394292196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,768,0.010696533322334289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,512,0.008012799918651581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,10240,0.10049173037211101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,512,0.017083734273910522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,256,0.004468266665935516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,7168,0.04990506569544474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,512,0.008108800152937572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,5120,0.04997760057449341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,256,0.006594133377075195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,256,0.01573013365268707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,128,0.003894400099913279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,128,0.005784533421198527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,64,0.003924266745646795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,128,0.01567893326282501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,32,0.00407679999868075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,64,0.01588159998257955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,12288,3072,0.03190186619758606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,32,0.015917866428693136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,51200,0.23477652867635093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,51200,0.22047680219014487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,65536,0.30389439264933266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,65536,0.48731520970662434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,51200,0.3838239987691244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,16384,0.13354239463806153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,16384,0.0886410633722941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,16384,0.08186879952748617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,12288,0.11946132977803547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,12288,0.06099199851353963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,10240,0.08728213310241699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,10240,0.05495573282241821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,10240,0.05305706659952799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,8192,0.07350719769795735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,8192,0.0473909338315328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,8192,0.043996798992156985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,7168,0.05601173241933187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,7168,0.04312746524810791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,7168,0.038318932056427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,5120,0.0406058669090271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,6144,0.03317653338114421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,5120,0.03540800015131633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,4096,0.033395198980967204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,4096,0.031750400861104325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,4096,0.024546132485071818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,3584,0.029598933458328248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,3584,0.029293866952260335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,3584,0.02244053284327189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,65536,0.2748138745625814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,3072,0.026099199056625368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,12288,2048,0.015995732943216958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,3072,0.027611732482910156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,2560,0.022563199202219643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,2560,0.024861866235733034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,2560,0.017543466885884602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,2048,0.01862293283144633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,12288,0.06982826391855876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,2048,0.023333332935969033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,2048,0.014760532975196838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,1536,0.014541866381963095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,1536,0.012301866213480632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,1536,0.021447465817133585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,1024,0.010696533322334289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,6144,0.0485642671585083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,1024,0.01914986570676168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,1024,0.010603732864061991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,768,0.009019733468691508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,768,0.009666132926940917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,6144,0.04013866583506266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,512,0.007223466535409291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,256,0.004106666644414266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,512,0.007646933197975159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,512,0.016081066926320393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,5120,0.02836693326632182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,128,0.00373333344856898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,256,0.0062730665008227035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,256,0.015618133544921874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,64,0.00344106654326121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,128,0.005251200000445048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,10240,32,0.0034986667335033415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,64,0.015324800213177999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,32,0.015362133582433065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,65536,0.21917972564697266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,65536,0.3929194768269857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,51200,0.3115647951761881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,12288,6144,0.045390931765238444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,768,0.01763413349787394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,16384,0.09525866508483886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,16384,0.06493866840998332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,10240,128,0.01520746648311615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,51200,0.20018347104390463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,16384,0.06572693188985189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,12288,0.07254079977671304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,12288,0.053707734743754065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,10240,0.060938668251037595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,10240,0.046758401393890384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,8192,0.04955093463261922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,8192,0.040742401281992593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,7168,0.04388159910837809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,7168,0.03722453514734904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,10240,3072,0.020032000541687012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,6144,0.03902399937311808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,51200,0.17745280265808105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,6144,0.034126933415730795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,5120,0.03282026648521423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,6144,0.02951893409093221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,5120,0.031197865804036457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,5120,0.02491626739501953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,4096,0.027079466978708906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,3584,0.024025599161783852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,4096,0.02132693330446879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,3584,0.025630933046340943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,3072,0.021280000607172646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,3072,0.023896533250808715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,3072,0.017117865880330405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,10240,0.04490240017573039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,2560,0.018242132663726807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,2560,0.022899200518925987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,2048,0.014985600113868713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,2048,0.021758933862050377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,2048,0.013645866513252258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,1536,0.01184213360150655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,1536,0.019848533471425376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,1536,0.011709866921106975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,1024,0.008921600381533305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,1024,0.01797653237978617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,1024,0.01032426655292511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,768,0.007534933090209961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,768,0.015954132874806723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,768,0.00902826686700185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,512,0.0060597335298856105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,512,0.015943466623624166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,512,0.007960533102353413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,3584,0.019681066274642944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,256,0.003930666546026865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,256,0.015256533026695251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,256,0.006684799989064534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,128,0.003568000098069509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,128,0.01511573294798533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,128,0.005715199808279673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,64,0.0033802665770053864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,8192,32,0.003532800078392029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,2560,0.015431466698646545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,32,0.015348266561826071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,12288,0.05101439952850342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,8192,0.03679146766662598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,65536,0.21018667221069337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,65536,0.34613653818766277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,7168,0.031777065992355344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,8192,65536,0.2566304047902425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,65536,0.2731562614440918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,16384,0.09067200024922689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,51200,0.17199145952860515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,12288,0.06437439918518066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,51200,0.2765429178873698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,16384,0.05959999958674113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,12288,0.049304533004760745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,16384,0.07181119918823242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,51200,0.22367893854777016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,4096,0.027612799406051637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,10240,0.043350398540496826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,10240,0.054131201903025304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,12288,0.0551584005355835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,8192,0.03932799895604451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,8192,0.037934935092926024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,7168,0.038898134231567384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,6144,0.03443520069122315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,7168,0.0347434679667155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,7168,0.03471680084864299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,6144,0.032321067651112874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,5120,0.02927253246307373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,5120,0.02914239962895711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,4096,0.024283732970555624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,5120,0.026259199778238936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,3584,0.021730132897694907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,4096,0.025988266865412397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,3584,0.024462932348251344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,3072,0.019131733973821004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,3072,0.023187200228373207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,3072,0.017811199029286705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,2560,0.01624853312969208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,2560,0.021983999013900756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,2560,0.01594239970048269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,2048,0.013345066706339517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,2048,0.020522665977478028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,2048,0.013385599851608277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,1536,0.010971732934316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,1536,0.01960853338241577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,8192,0.046743468443552656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,1024,0.008269866804281871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,6144,0.029534933964411418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,1024,0.017248000701268515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,8192,64,0.01548693378766378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,768,0.007165866593519847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,768,0.008922666311264038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,512,0.004670933385690053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,512,0.015554133057594299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,256,0.0037685332198937735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,1024,0.009574400385220845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,256,0.01518186628818512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,768,0.015762133399645488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,256,0.0059914668401082356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,128,0.0034133332471052804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,128,0.005257600049177805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,64,0.0032330666979153953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,128,0.015012266238530478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,7168,32,0.003305600086847941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,64,0.015106133619944253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,4096,0.02170026699701945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,3584,0.019756799936294554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,65536,0.2919114748636881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,65536,0.1769418716430664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,65536,0.2693290710449219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,16384,0.08162986437479655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,51200,0.23194452921549477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,16384,0.055070932706197104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,51200,0.1424608071645101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,16384,0.07049173514048258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,12288,0.06497173309326172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,12288,0.04718826611836751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,12288,0.052408532301584876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,10240,0.05486720005671183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,10240,0.049559466044108075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,10240,0.03972906668980916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,8192,0.03977280060450236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,1536,0.01111893355846405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,8192,0.03887786865234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,7168,0.03471999963124593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,8192,0.0358240008354187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,6144,0.031065599123636885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,6144,0.030422399441401165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,6144,0.028755199909210206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,5120,0.026143999894460042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,5120,0.027445334196090698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,5120,0.02444480061531067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,7168,32,0.015190399686495461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,4096,0.02200746734937032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,4096,0.025293866793314617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,4096,0.021342933177947998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,3584,0.019522132476170857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,3072,0.01717653274536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,3584,0.0191594660282135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,3584,0.023682133356730143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,2560,0.014866133530934652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,3072,0.022421334187189737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,2560,0.015310933192571005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,2560,0.021277866760889688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,2048,0.012472533186276754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,7168,512,0.007369600236415863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,2048,0.012938666343688964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,1536,0.010212266445159912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,1536,0.01853013237317403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,1536,0.010557867089907328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,1024,0.007821866869926452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,1024,0.015595733126004537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,7168,0.03266666730244954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,768,0.006726400057474773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,768,0.016028799613316855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,768,0.008188800017038981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,512,0.004323199888070424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,512,0.015629866719245912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,512,0.00720000018676122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,256,0.0037632000943024955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,3072,0.0172437330087026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,256,0.015158399939537048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,128,0.0034133332471052804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,2048,0.020403200387954713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,128,0.005142400165398916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,64,0.0032437334458033243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,64,0.014999467134475707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,6144,32,0.003335466732581457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,32,0.015134933590888976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,51200,0.21275839805603028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,65536,0.2512426694234212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,10240,0.04679146607716878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,7168,0.03335786660512288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,65536,0.2648277282714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,256,0.0059456000725428265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,51200,0.19956266085306804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,51200,0.12179733117421468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,16384,0.06829439798990886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,6144,128,0.015220266580581666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,16384,0.0535370667775472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,51200,0.21082986195882164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,12288,0.040380799770355226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,12288,0.050964268048604336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,10240,0.04647466739018758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,10240,0.03589333295822143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,8192,0.03862080176671346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,8192,0.032150399684906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,8192,0.03844053347905477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,7168,0.02905813256899516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,7168,0.02932800054550171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,6144,1024,0.008755200107892354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,6144,0.025864533583323163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,6144,0.027046400308609008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,5120,0.022125866015752158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,6144,0.02877333362897237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,4096,0.018323200941085815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,5120,0.02463679909706116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,5120,0.025050665934880572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,4096,0.023514666159947715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,3584,0.016145066420237223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,3584,0.018564265966415406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,3584,0.022129066785176597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,3072,0.014265599846839904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,3072,0.021117866039276123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,3072,0.016910932461420693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,65536,0.15827840169270832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,2560,0.012322133779525757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,2048,0.010406399766604107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,2560,0.020054399967193604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,2560,0.015171200037002563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,1536,0.008692266543706258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,2048,0.019352533419926963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,1536,0.010228266318639118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,1536,0.017862399419148765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,1024,0.006925866504510244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,1024,0.015948800245920818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,768,0.005715199808279673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,768,0.01609280010064443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,12288,0.0535584012667338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,512,0.004058666775623957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,512,0.01555519998073578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,16384,0.06943786938985189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,256,0.0035989334185918174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,256,0.015079466501871744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,10240,0.04471360047658284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,128,0.003257599969704946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,128,0.014929067095120749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,7168,0.03172053297360738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,128,0.005081599950790406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,64,0.003020799905061722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,64,0.014966400464375815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,5120,32,0.0030954666435718536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,5120,32,0.01513706644376119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,4096,0.020707199970881142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,65536,0.19583360354105633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,65536,0.12754240036010742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,2048,0.01267413298288981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,51200,0.15569705963134767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,65536,0.2250826676686605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,51200,0.10300693511962891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,16384,0.05322773456573486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,1024,0.008605866630872091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,16384,0.04480746587117513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,12288,0.04218026796976725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,12288,0.03812053203582764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,10240,0.036246399084726974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,768,0.008228266735871632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,10240,0.03240639964739482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,10240,0.03836693366368611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,8192,0.03032853404680888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,8192,0.02858133316040039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,7168,0.027486934264500933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,7168,0.026652799050013228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,512,0.007159466544787089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,6144,0.02455893357594808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,6144,0.025084799528121947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,6144,0.02359893321990967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,5120,0.01877440015474955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,5120,0.023171200354894003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,5120,0.02020053267478943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,4096,0.015408000349998474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,4096,0.021733333667119346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,4096,0.01709973414738973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,3584,0.013782399892807006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,3584,0.02078826626141866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,5120,256,0.0059456000725428265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,3072,0.01221440037091573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,3072,0.01990506649017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,51200,0.1641983985900879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,16384,0.0560149351755778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,2560,0.01148373285929362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,2560,0.019569067160288493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,12288,0.04184960126876831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,2048,0.009271466732025146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,8192,0.031325866778691605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,2048,0.011821867028872172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,2048,0.018833067019780478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,1536,0.007799466451009114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,1024,0.00621973325808843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,1536,0.01651946703592936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,1024,0.0156960000594457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,768,0.004478933413823446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,7168,0.026476800441741943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,768,0.015703466534614564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,512,0.0039061332742373147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,512,0.015326933066050211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,512,0.0071733335653940845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,256,0.0034783999125162757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,256,0.014963199694951376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,128,0.003126399964094162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,256,0.006035199761390686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,128,0.014731733004252115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,128,0.005103999873002371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,64,0.002961066613594691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,4096,32,0.003143466760714849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,64,0.014811733365058899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,4096,32,0.015171200037002563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,3584,0.016309332847595216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,65536,0.1833024024963379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,65536,0.12087893486022949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,65536,0.20829440752665201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,51200,0.14568959871927897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,51200,0.09660800298055014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,16384,0.04249279896418254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,16384,0.05132266680399576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,51200,0.16560746828715006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,16384,0.05580373207728068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,12288,0.038520534833272294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,10240,0.03321386575698852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,12288,0.04245866537094116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,12288,0.03546346823374431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,8192,0.027353600660959883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,8192,0.026594134171803792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,3072,0.014437333742777506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,7168,0.02469866673151652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,7168,0.025034666061401367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,7168,0.02765866716702779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,6144,0.022127999862035116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,6144,0.023780266443888344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,2560,0.013264000415802002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,5120,0.01657919983069102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,5120,0.022642133633295695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,10240,0.030217599868774415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,1536,0.009383466839790345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,4096,0.013941333691279093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,1024,0.008389332890510559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,4096,0.021128533283869426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,3584,0.012283733487129212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,3584,0.020390399297078452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,4096,768,0.008172800143559773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,3072,0.011058132847150166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,3072,0.019803732633590698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,2560,0.009778133034706116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,2560,0.01890666683514913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,10240,0.03643733263015747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,2048,0.008566400408744812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,2048,0.01731626590092977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,8192,0.031677865982055665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,2048,0.011086933811505635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,1536,0.007494399944941203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,1536,0.009369599819183349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,1536,0.016173866391181946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,1024,0.005190399785836538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,1024,0.008102400104204814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,1024,0.016013866662979125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,768,0.004653866589069367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,768,0.01581333378950755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,768,0.007788800199826558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,512,0.004037333279848098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,512,0.006971733272075653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,512,0.015244799852371215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,256,0.003492266684770584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,256,0.01492800017197927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,6144,0.02442453304926554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,128,0.0031221332649389905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,128,0.014691199858983359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,128,0.005027199784914652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,64,0.002916266769170761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,64,0.014806399742762247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3584,32,0.002993066608905792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3584,32,0.014840533336003622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,5120,0.021868799130121866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,65536,0.15364480018615723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,4096,0.017642666896184288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,65536,0.10566293398539225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,3584,0.01664426624774933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,3072,0.01474133332570394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,51200,0.12372266451517741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,51200,0.08484586874643961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,16384,0.0441269318262736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,51200,0.19433172543843585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,16384,0.03910826841990153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,16384,0.06539306640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,12288,0.03365866740544637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,2560,0.01317759950955709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,12288,0.03318933248519897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,10240,0.02884693344434102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,12288,0.04822506507237752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,10240,0.02969599962234497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,8192,0.024642133712768556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,8192,0.03504213492075602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,7168,0.021966934204101562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,8192,0.02555519938468933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,7168,0.02411093314488729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,6144,0.019686400890350342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,7168,0.030588799715042116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,6144,0.023026132583618165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,5120,0.015597866972287497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,5120,0.02141973376274109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,4096,0.012894933422406515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,4096,0.02057386636734009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,4096,0.019885865847269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,3584,0.01143999993801117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,3584,0.01996799906094869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,3072,0.010188800096511842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,3072,0.01919040083885193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,3072,0.015947733322779337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,2560,0.009152000149091084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,2560,0.017946666479110716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,2560,0.013994666934013366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,2048,0.008111999928951263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,2048,0.016502400239308677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,1536,0.0070816000302632645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,1536,0.016237866878509522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,1536,0.009483733773231506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,1024,0.004724266628424326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,1024,0.016294399897257485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,1024,0.008262399832407634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,768,0.004613333443800608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,768,0.015878400206565856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,768,0.008026666442553202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,512,0.004098133246103922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,512,0.015363199512163797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,65536,0.24245227177937828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,256,0.0033919999996821085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,256,0.015084800124168397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,10240,0.04185386498769124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,256,0.005704533557097117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,128,0.0030954666435718536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,128,0.01472106675306956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,128,0.004891733328501383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,64,0.0029205332199732465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,3072,32,0.0029834667841593427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,6144,0.02741439938545227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,64,0.014745600024859109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,3072,32,0.014909866452217101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,5120,0.02285760045051575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,65536,0.09251946608225504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,3584,0.018074667453765868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,51200,0.10569813251495361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,51200,0.07309546470642089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,16384,0.03554133176803589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,16384,0.034728535016377765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,51200,0.17400746345520018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,12288,0.02841493288675944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,12288,0.029521065950393676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,12288,0.045252267519632974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,10240,0.024829866488774617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,10240,0.02706986665725708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,8192,0.02070080041885376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,2048,0.011577600240707397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,8192,0.024552534023920693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,8192,0.03479466835657756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,7168,0.018684800465901694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,7168,0.02397759954134623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,6144,0.016427733500798545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,6144,0.023201066255569457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,6144,0.02615999976793925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,5120,0.014340266585350037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3584,256,0.0058783998092015585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,5120,0.02099840044975281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,5120,0.02191466689109802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,4096,0.01120853324731191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,65536,0.13224213123321532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,4096,0.020070399840672812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,4096,0.018793600797653198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,3584,0.011703466375668842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,3584,0.017056000232696534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,3584,0.01954666574796041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,3072,0.010342400272687275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,3072,0.018326399723688762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,2560,0.008533333738644917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,2560,0.017015467087427773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,2560,0.013166933258374532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,2048,0.007416533430417378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,2048,0.0172650674978892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,10240,0.041530664761861166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,2048,0.01074026624361674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,1536,0.005960533519585928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,1536,0.016593066851298015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,3072,512,0.007144533097743988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,1024,0.01589120030403137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,1024,0.008155733346939087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,768,0.004205866654713949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,768,0.015401599804560342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,768,0.007460266848405202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,512,0.003769599894682566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,512,0.015290666619936624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,512,0.006647466619809468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,256,0.0034933333595593774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,256,0.014843733112017313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,256,0.005599999924500784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,128,0.0031317333380381264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,128,0.014616533120473226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,128,0.005015466610590617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,64,0.0029472000896930695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,64,0.014756266276041666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,32,0.003020799905061722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,65536,0.22259947458902993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2560,32,0.014728533228238425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,16384,0.06106453339258829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,65536,0.1042453368504842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2560,1024,0.004786133269468943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,65536,0.07719573179880777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,51200,0.07996799945831298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,51200,0.0639520009358724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,65536,0.20439146359761556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,16384,0.029734400908152263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,51200,0.15716266632080078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,16384,0.031455999612808226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,12288,0.02385279933611552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,16384,0.05493866602579752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,10240,0.020717867215474448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,12288,0.03867093324661255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,8192,0.017106133699417114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,10240,0.025227733453114826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,8192,0.030011733373006184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,8192,0.02336533268292745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,7168,0.015455999970436096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,7168,0.02237866719563802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,6144,0.014087466398874917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,6144,0.021522132555643718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,3072,0.015172266960144043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,5120,0.011851732929547627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,5120,0.020594133933385213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,5120,0.01909760038057963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,4096,0.010103467106819152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,4096,0.019633066654205323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,4096,0.016438399751981102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,3584,0.00943893293539683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,3584,0.014854400356610616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,3584,0.01874986688296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,3072,0.008706133564313252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,1536,0.009291733304659527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,3072,0.013313066959381104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,3072,0.017245866854985557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,2560,0.007812266548474629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,12288,0.02707306742668152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,2048,0.00710399995247523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,2560,0.0169706662495931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,2560,0.012062933047612507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,2048,0.016689066092173258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,2048,0.010180266698201497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,1024,0.004322133461634318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,1536,0.00901759962240855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,1536,0.01630506714185079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,768,0.003964799890915553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,1024,0.00813973347345988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,1024,0.015482667088508605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,768,0.00783679982026418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,768,0.015681067109107973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,512,0.0036469332873821257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,256,0.003236266722281774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,512,0.006896000107129414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,512,0.015140266219774882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,256,0.005832533538341522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,256,0.01471573313077291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,128,0.003081600119670232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,64,0.0029397333661715193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,128,0.014969600240389505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,32,0.002994133283694585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,64,0.0146506667137146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,2048,32,0.014577066898345948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,65536,0.08120960394541422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,65536,0.0676746686299642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2560,7168,0.028963200251261395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,2048,1536,0.0050240000089009605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,51200,0.055497598648071286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,51200,0.06517653465270996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,16384,0.024215465784072875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,16384,0.02714346647262573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,51200,0.14594346682230633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,12288,0.019372800985972084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,7168,0.02469759980837504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,128,0.004889599978923798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,12288,0.025603200991948443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,12288,0.0377461314201355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,10240,0.01653439998626709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,10240,0.024246400594711302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,8192,0.013758933544158936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,8192,0.02276479999224345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,8192,0.027126399676005046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,7168,0.012662399808565775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,7168,0.02137706677118937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,6144,0.022566399971644082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,6144,0.012309333682060242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,6144,0.02136639952659607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,5120,0.011018666625022887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,5120,0.02024959921836853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,2048,10240,0.03687146504720052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,4096,0.009475200374921163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,4096,0.0179584006468455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,3584,0.00886079967021942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,3584,0.0173962672551473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,65536,0.18032852808634442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,3072,0.00812906672557195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,3072,0.017388800779978432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,3072,0.013033599654833476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,2560,0.006922666728496551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,2560,0.01686613361040751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,2560,0.01121493379275004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,2048,0.0060245335102081295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,2048,0.016522666811943053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,2048,0.009834667046864826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,1536,0.005100800096988678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,1536,0.01580586632092794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,16384,0.050215466817220056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,1024,0.004293333490689596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,10240,0.03323413332303365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,1024,0.007758933305740357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,1024,0.015596800049146018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,768,0.004020266731580098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,768,0.015397333105405173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,512,0.003735466549793879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,7168,0.023939200242360435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,512,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,512,0.006289066871007283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,256,0.003202133377393087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,256,0.014852266510327658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,256,0.005291733145713806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,128,0.0029728000362714132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,128,0.014699733257293702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,64,0.002833066632350286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,5120,0.01903146704037984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,64,0.014787200093269347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1536,32,0.002880000074704488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1536,32,0.014850133657455444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,65536,0.054474667708079016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,65536,0.053795198599497475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,51200,0.043900799751281736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,65536,0.13896959622701008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,51200,0.045254401365915936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,51200,0.11103359858194987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,16384,0.023685334126154582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,16384,0.0425493319829305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,12288,0.013826133807500205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,12288,0.02286400000254313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,4096,0.01578133304913839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,10240,0.012226133545239767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,12288,0.031897600491841635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,10240,0.029959466060002642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,10240,0.021313067277272543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,8192,0.010825600226720173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,8192,0.020716800292332967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,3584,0.013790933291117349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,7168,0.010027733445167542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,6144,0.020683733622233073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,7168,0.021699200073877968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,7168,0.020173867543538414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,6144,0.010134399930636088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,6144,0.018782933553059898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,6144,0.02053119937578837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,5120,0.009417600433031718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,4096,0.008391466736793519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,5120,0.016396799683570863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,5120,0.019205333789189656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,4096,0.01344319979349772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,3584,0.00798826664686203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,4096,0.018702934185663857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,3072,0.008029866715272267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,3584,0.011997866630554199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,3584,0.018179200092951455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,3072,0.01067200005054474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,3072,0.01723519961039225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,2560,0.007289599875609081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,2560,0.016566399733225504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,2560,0.010021332899729412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,2048,0.006566399832566579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,2048,0.01646506687005361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,2048,0.008310399949550629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,16384,0.01743040084838867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,1536,0.005147733290990194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,1024,0.0044821331898371375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,1536,0.016008533040682473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,1536,0.007844266792138416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,768,0.004090666770935059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,1024,0.006857599814732869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,1024,0.015505066514015198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,512,0.0036330667634805044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,768,0.006198399762312571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,512,0.005757866799831391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,512,0.014989866813023885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,256,0.0031701333820819853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,256,0.01470186710357666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,256,0.005016533533732096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,128,0.0029002666473388673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,128,0.0048213332891464235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,128,0.014465066790580749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,64,0.0028053333361943563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,1024,32,0.0028031999866167706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,1536,0.008703999718030294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1024,8192,0.02501759926478068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,32,0.014679466684659323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,65536,0.04351679881413777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,51200,0.035419734319051106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,65536,0.04788159926732381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,768,0.007170133292675018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,65536,0.12593599955240886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,16384,0.014082133769989014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,51200,0.10072746276855468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,16384,0.03673386573791504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,12288,0.011667199929555257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,12288,0.02813439965248108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,12288,0.022167466084162393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,1536,128,0.004817066589991251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,10240,0.010889599720637005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,10240,0.020820266008377074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,8192,0.010964266459147136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,8192,0.019826134045918785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,7168,0.010331733028093974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,7168,0.01904426614443461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,7168,0.01770026683807373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,6144,0.009609599908192951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,6144,0.018601600329081217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,768,0.015581867098808289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,6144,0.015745066603024802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,5120,0.008759466807047527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,5120,0.019014400243759156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,4096,0.007514666517575581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,4096,0.018054399887720743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,1024,64,0.01444586714108785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,3584,0.007028266787528992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,3584,0.01755626598993937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,3584,0.009783466657002766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,3072,0.006512000163396199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,51200,0.045864534378051755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,16384,0.023655466238657632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,3072,0.017038933436075845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,2560,0.00628053347269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,2560,0.016588800152142844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,2560,0.008728532989819845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,2048,0.005444266895453135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,2048,0.007294933497905731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,1536,0.0048543999592463175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,8192,0.020540799697240195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,1536,0.007111466427644093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,1024,0.004178133110205332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,1024,0.015631999572118124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,768,0.003849600007136663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,768,0.015410133202870688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,768,0.006050133208433787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,512,0.0034261333445707956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,512,0.015013333161671957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,512,0.005508266886075338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,256,0.0031914666295051576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,256,0.014665599664052328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,256,0.005041066805521647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,128,0.0029898665845394133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,128,0.014538666605949402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,128,0.004804266492525736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,64,0.0027776000400384264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,3072,0.009392000238100688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,64,0.014552533626556396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,768,32,0.0028575999041398365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,32,0.0146506667137146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,65536,0.03173333406448364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,2048,0.016361600160598753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,65536,0.04093653361002604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,51200,0.026226133108139038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,768,1536,0.015656532843907674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,16384,0.01232319970925649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,51200,0.08177706400553385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,10240,0.026115200916926068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,16384,0.02190399964650472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,12288,0.00992746651172638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,16384,0.028948267300923664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,1024,0.006217599908510844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,12288,0.022000000874201456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,12288,0.02033066749572754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,10240,0.010066133737564088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,10240,0.01843520005544027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,10240,0.019130667050679527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,8192,0.015511467059453329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,8192,0.01858773430188497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,7168,0.00832426647345225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,7168,0.019090133905410766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,7168,0.013492266337076822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,6144,0.007527466615041096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,6144,0.011916800340016683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,6144,0.019410133361816406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,5120,0.006794666747252147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,5120,0.01853440006573995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,4096,0.00628053347269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,5120,0.013289599617322286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,4096,0.01804800033569336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,4096,0.00941973328590393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,3584,0.006687999765078227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,768,4096,0.011370666821797689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,3584,0.009060266613960265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,3584,0.017438934246699015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,3072,0.006856533388296763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,2560,0.006066133578618368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,3072,0.00830506682395935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,3072,0.01698026657104492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,2048,0.00537066658337911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,2560,0.007830399771531422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,2560,0.01648640036582947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,1536,0.004744533201058706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,2048,0.0069013332327206925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,2048,0.01604373355706533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,1024,0.004133333265781402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,1536,0.015795200069745382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,65536,0.10554666519165039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,1024,0.015132799744606018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,51200,0.03430720170338948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,768,0.015305599570274353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,768,0.005487999816735586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,512,0.0034048000971476236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,512,0.015255467096964518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,256,0.003176533430814743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,256,0.01470186710357666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,8192,0.00902400016784668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,128,0.002881066749493281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,128,0.01453439990679423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,128,0.004609066744645437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,64,0.0028053333361943563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,64,0.01458133359750112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,32,0.0027722666660944624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,512,32,0.01455466647942861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,65536,0.019871999820073448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,65536,0.032278400659561154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,65536,0.08750933011372884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,51200,0.015378133455912272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,1536,0.006687999765078227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,51200,0.028282666206359865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,1024,0.005804799993832906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,512,768,0.0038399999340375268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,16384,0.009596799810727436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,51200,0.06798400084177653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,12288,0.007782400151093801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,16384,0.02329813241958618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,12288,0.01918399930000305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,12288,0.01899626652399699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,512,0.005268266797065735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,8192,0.006558933357397716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,10240,0.01935466726620992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,8192,0.014082133769989014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,8192,0.018331732352574667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,7168,0.006342400113741558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,7168,0.018772266308466592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,7168,0.01274133324623108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,6144,0.006312533219655354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,6144,0.018580265839894614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,6144,0.011643733580907185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,5120,0.006528000036875407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,4096,0.006108800073464712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,5120,0.010363733768463135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,5120,0.018934400876363118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,3584,0.006589866677920024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,4096,0.0093642661968867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,4096,0.017832533518473307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,3584,0.009090133508046468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,3072,0.006230400005976359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,3584,0.01744426687558492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,2560,0.006086400151252747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,3072,0.017018665870030723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,2560,0.007717333237330119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,2560,0.016532267133394875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,2048,0.0054175997773806255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,2048,0.016152532895406087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,2048,0.0069248000780741375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,1536,0.004752000172932943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,1536,0.015778133273124696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,1536,0.006687999765078227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,5120,0.010607999563217164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,1024,0.0040287998815377556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,1024,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,1024,0.005709866682688395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,768,0.003732266773780187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,768,0.015346133708953857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,768,0.005322666466236114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,512,0.0034304000437259674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,512,0.015042133132616677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,256,0.0030229332546393077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,16384,0.019817600647608437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,256,0.014747732877731323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,10240,0.007092266778151194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,128,0.002808533360560735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,256,0.004808533191680908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,10240,0.016770132382710776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,64,0.0027637332677841187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,128,0.0045952002207438145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,256,32,0.002784000088771184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,64,0.014405333002408347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,32,0.014691199858983359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,65536,0.011710932850837708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,51200,0.010268800457318624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,65536,0.028198399146397907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,65536,0.08431466420491537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,16384,0.0064074665307998655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,51200,0.024923733870188394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,51200,0.06736640135447183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,16384,0.020205867290496827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,12288,0.006904533505439759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,16384,0.023515733083089192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,12288,0.019219199816385903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,12288,0.019105066855748497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,512,256,0.004932266473770141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,8192,0.006716800232728322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,10240,0.01877546707789103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,8192,0.014220800002415976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,8192,0.018398932615915933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,7168,0.006356266637643178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,7168,0.01874666611353556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,7168,0.012712533275286356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,6144,0.006055466830730438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,6144,0.011736533045768738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,6144,0.018283732732137046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,5120,0.006457599997520447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,5120,0.01874666611353556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,5120,0.010433066884676616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,4096,0.006001066664854685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,4096,0.009303466478983561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,4096,0.017758933703104655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,3584,0.006419200201829274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,3584,0.01750826636950175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,3072,0.006108800073464712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,3072,0.017169066270192466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,3072,0.008109866579373678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,2560,0.006135466694831848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,256,128,0.014550399780273438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,2560,0.01647040049235026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,2560,0.0077450667818387345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,2048,0.005459199845790863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,2048,0.006917333106199901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,2048,0.016155733664830526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,1536,0.004702933132648468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,1536,0.01571626663208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,1536,0.006697600086530049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,1024,0.004037333279848098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,1024,0.015531733632087708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,1024,0.0058794667323430385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,768,0.003717333326737086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,10240,0.006451199948787689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,768,0.005437866846720377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,768,0.015558399756749473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,512,0.003457066665093104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,512,0.014919466773668923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,256,0.003070933371782303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,256,0.004795733094215393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,256,0.015026133259137472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,128,0.0028768000503381092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,128,0.014535466829935709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,128,0.004604800045490265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,64,0.0027445333699385325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,128,32,0.0026975999275843303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,64,0.01451520025730133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,3072,0.00807360013326009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,128,32,0.014545067151387533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,65536,0.010069333513577779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,51200,0.008482133348782856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,65536,0.026898133754730224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,16384,0.0067552000284194945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,51200,0.023893332481384276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,16384,0.019215999046961467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,10240,0.006232533355553945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,12288,0.018502400318781535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,10240,0.019381332397460937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,8192,0.006444799900054932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,7168,0.006046933432420095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,8192,0.018450133005777993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,6144,0.005869866907596588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,7168,0.01834239959716797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,5120,0.006303999821345012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,6144,0.01838399966557821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,4096,0.00591893345117569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,4096,0.017963733275731406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,3584,0.006306133170922597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,3584,0.017237333456675212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,3072,0.005950933198134104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,3072,0.016922666629155477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,2560,0.005959466596444448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,2560,0.016378666957219443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,2048,0.005334400137265523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,512,0.005157333115736643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,1536,0.004664533336957296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,2048,0.01595093309879303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,1024,0.003949866692225138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,1536,0.015617066621780395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,768,0.0036746665835380556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,1024,0.01529813309510549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,512,0.0033621333539485933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,768,0.015304533640543619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,256,0.0030239999294281008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,512,0.014668800433476768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,128,0.0028437333802382152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,256,0.014507733782132468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,64,0.002700799951950709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,128,0.014544000228246054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,32,0.0027189334233601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,64,0.014526933431625366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,32,0.014333867033322654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,65536,0.011542399724324543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,51200,0.00960640013217926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,65536,0.026387200752894087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,16384,0.007328000168005626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,51200,0.026316799720128375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,16384,0.019054933389027914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,12288,0.006029866635799408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,64,5120,0.018579200903574625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,10240,0.006369066735108693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,12288,0.01877973278363546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,8192,0.006085333228111267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,10240,0.018911999464035035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,7168,0.006107733150323232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,8192,0.01807039976119995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,6144,0.005880533158779145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,7168,0.018245333433151247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,5120,0.006253866851329804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,6144,0.018175999323527016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,4096,0.005854933460553487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,5120,0.01855573256810506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,3584,0.006212266782919565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,4096,0.017413334051767985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,3072,0.0059797331690788266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,3584,0.017195733388264973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,2560,0.00600853314002355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,256,512,0.005067733426888784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,2048,0.005342933535575867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,2560,0.016927999258041383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,1536,0.004732800026734671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,2048,0.01618666648864746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,1024,0.004018133382002512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,1536,0.015532799561818442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,768,0.0036650667587916053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,1024,0.01569706698258718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,512,0.003573333223660787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,768,0.015064533551534018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,256,0.0030591999491055804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,512,0.014846932888031007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,128,0.0028586665789286296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,256,0.014657066265741984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,64,0.002664533257484436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,128,0.014513066411018372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,32,32,0.0027061333258946735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,32,0.01454080045223236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,65,64,12288,0.0060576001803080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,3072,0.016590933005015053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,10240,0.016790399948755898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,65,32,64,0.014640000462532044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,16384,0.4059573491414388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,12288,0.5903690973917644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,16384,0.7739477157592773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,51200,1.2375754038492839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,65,128,3584,0.009010133147239686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,12288,0.31120640436808267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,10240,0.4919541358947754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,10240,0.3083360036214193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,8192,0.39198185602823893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,51200,2.437182871500651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,8192,0.21708480517069498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,7168,0.34493974049886067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,7168,0.2220128059387207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,6144,0.29690240224202474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,6144,0.1929375966389974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,5120,0.24887572924296059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,5120,0.14214186668395995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,16384,0.3678741455078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,4096,0.2022069295247396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,51200,1.1203914642333985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,4096,0.11627519925435384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,12288,0.27865174611409504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,3584,0.17804053624471028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,10240,0.23533867200215658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,3584,0.10566826661427815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,8192,0.1896245320638021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,3072,0.15295467376708985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,3072,0.09670399824778239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,2560,0.12900479634602863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,7168,0.16712853113810222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,2560,0.08125759760538737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,2048,0.10276906490325928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,2048,0.06778240203857422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,6144,0.1453418731689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,1536,0.08730773131052652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,1536,0.05690666834513346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,5120,0.12286506493886311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,1024,0.06339413324991862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,1024,0.04353173176447551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,768,0.04437226851781209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,768,0.038381866614023846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,4096,0.1137664000193278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,512,0.03524800141652425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,512,0.0328821341196696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,3072,0.07801386515299479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,256,0.020240000883738198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,256,0.022411733865737915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,768,0.02650880018870036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,128,0.01048959990342458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,128,0.01996799906094869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,2048,0.05490560134251913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,64,0.008611200253168742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,64,0.017679999272028603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,32,0.006413866579532623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,32,0.01844266653060913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,1024,0.032374399900436404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,3584,0.08907732963562012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,2560,0.06841493447621663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,65536,1.2376363118489584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,65536,2.36985600789388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,128,0.012365866700808208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,256,0.014345600207646688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,51200,1.9009930928548175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,51200,0.9722378412882486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,512,0.020823466777801513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,1536,0.04312426646550496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,16384,0.3204533259073893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,16384,0.5966463724772135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,12288,0.490173880259196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,12288,0.24514880180358886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,10240,0.3835839907328288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,10240,0.2188426653544108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,8192,0.17115306854248047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,8192,0.306440544128418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,7168,0.2652714729309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,7168,0.15016959508260092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,6144,0.2291872024536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,6144,0.13285333315531414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,65536,1.1321301778157553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,5120,0.22475733757019042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,12288,0.2213482697804769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,5120,0.13033706347147625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,4096,0.15653653144836427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,4096,0.09516159693400064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,16384,0.33946558634440105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,3584,0.1596949259440104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,10240,0.21390612920125326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,3584,0.09428693453470866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,51200,0.9369802474975586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,3072,0.12131306330362956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,6144,0.11484906673431397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,3072,0.08614719708760579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,2560,0.10020693143208821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,2560,0.07409066359202067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,2048,0.09386666615804037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,7168,0.13247360388437907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,2048,0.06319253444671631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,1536,0.06355946858723958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,8192,0.15093119939168292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,1536,0.04756480058034261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,1024,0.044912000497182206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,1024,0.03761386473973592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,3072,0.06196586688359579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,768,0.03978879849116008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,768,0.03208960096041362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,4096,0.08046080271402994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,512,0.028655999898910524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,512,0.02577280004819234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,1536,0.0389301339785258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,256,0.013801599542299906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,256,0.02345386743545532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,5120,0.09743573665618896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,128,0.0113045334815979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,128,0.020423465967178346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,2560,0.053761065006256104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,64,0.007670400043328603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,64,0.019565866390864054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,51200,32,0.007484800120194752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,51200,32,0.019849600394566853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,3584,0.07681813240051269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,1024,0.026572799682617186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,2048,0.04408106803894043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,65536,0.7540789286295573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,65536,0.42418880462646485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,51200,0.5953952153523763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,51200,0.32709760665893556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,16384,0.1921770731608073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,16384,0.11814613342285156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,256,0.01260373294353485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,12288,0.14844266573588055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,128,0.009923199812571209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,12288,0.09104959964752198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,10240,0.12472000122070312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,10240,0.07770559787750245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,8192,0.09887573719024659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,8192,0.0670634667078654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,512,0.016661333044370015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,7168,0.0864362637201945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,7168,0.060602664947509766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,51200,768,0.022090667486190797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,6144,0.07572906812032064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,6144,0.055105066299438475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,65536,0.3692277272542318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,5120,0.07232106526692708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,5120,0.047780267397562665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,51200,0.2924288113911947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,4096,0.05870186487833658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,16384,0.098854398727417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,4096,0.041501867771148684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,3584,0.04492799838383992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,3584,0.0381717324256897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,12288,0.07597866853078207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,3072,0.03931946754455566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,3072,0.034289065996805826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,10240,0.0646506667137146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,2560,0.0335477352142334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,2560,0.03134186665217082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,8192,0.05324586629867554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,2048,0.027305599053700762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,2048,0.02786453366279602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,7168,0.04736959934234619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,1536,0.02173653244972229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,1536,0.02442880074183146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,6144,0.04185386498769124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,1024,0.01532373329003652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,1024,0.020980266729990642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,5120,0.03601813316345215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,768,0.011954133709271748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,768,0.019607466459274293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,4096,0.030280532439549764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,512,0.009062400460243225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,512,0.018016000588734947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,3072,0.024522666136423746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,256,0.006019199887911478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,256,0.015758933623631795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,2048,0.017977599302927652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,128,0.003964799890915553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,128,0.015427199999491372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,1024,0.011664000153541566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,64,0.0037429332733154297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,64,0.015326933066050211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,32,0.003884800026814143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,32,0.015479466319084168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,65536,0.5842517217000325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,3584,0.027560534079869588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,65536,0.3180405298868815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,2560,0.021647999684015908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,51200,0.23926080067952477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,51200,0.45278078715006514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,16384,0.16589120229085286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,16384,0.08862293561299642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,512,0.008763733506202697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,12288,0.10733226935068767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,768,0.010345600048700969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,128,0.005840000013510386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,256,0.006154666841030121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,12288,0.07077866395314535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,10240,0.0898080031077067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,10240,0.06197973489761353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,8192,0.07281813621520997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,1536,0.014923733472824097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,7168,0.06414186557133993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,7168,0.04771733283996582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,8192,0.052475734551747644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,6144,0.05607680082321167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,6144,0.043866666158040364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,65536,0.28216425577799475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,5120,0.05438079833984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,5120,0.038534398873647055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,16384,0.07632746696472167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,4096,0.03879893223444621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,12288,0.05894720156987508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,4096,0.03386773268381755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,3584,0.0342741330464681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,3584,0.031829333305358885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,51200,0.22190292676289877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,3072,0.034399998188018796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,3072,0.029478400945663452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,10240,0.05026880105336508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,2560,0.0256223996480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,2560,0.026713599761327107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,6144,0.03286186655362447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,7168,0.03717439969380697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,2048,0.021126399437586464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,2048,0.023898667097091673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,8192,0.041740798950195314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,1536,0.016739199558893837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,1536,0.021794132391611733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,1024,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,1024,0.01949866612752279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,3072,0.01935466726620992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,768,0.009545600414276123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,768,0.01844373345375061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,4096,0.02400319973627726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,5120,0.028602667649586994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,512,0.007437866429487865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,512,0.015319466590881348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,256,0.004125866790612539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,256,0.015309866269429526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,3584,0.02172586719195048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,128,0.003753600021203359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,128,0.015175466736157736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,1536,0.011932800213495891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,64,0.0033632000287373864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,64,0.015217066804567973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,32,0.003542399903138479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,32,0.015335466464360556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,2560,0.017107200622558594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,65536,0.4821173350016276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,65536,0.27855892181396485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,2048,0.014337066809336343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,51200,0.38284692764282224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,51200,0.22053759892781577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,16384,0.11637226740519206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,16384,0.07590719858805338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,1024,0.009898666540781658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,12288,0.08882986704508464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,12288,0.06178346474965414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,256,0.005553066730499268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,10240,0.07446186542510987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,128,0.005184000233809153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,10240,0.054067198435465494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,8192,0.06086613337198893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,8192,0.04656853278477986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,512,0.006140799820423126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,7168,0.05341226657231649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,7168,0.042379732926686606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,768,0.008689066767692566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,6144,0.047093331813812256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,6144,0.0389024019241333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,65536,0.2392021338144938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,5120,0.04541973272959392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,5120,0.034833065668741864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,16384,0.06525760094324748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,4096,0.032525867223739624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,4096,0.031464533011118574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,51200,0.19070399602254232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,12288,0.05078186591466268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,3584,0.03304319977760315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,3584,0.02885226607322693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,3072,0.025417600075403852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,3072,0.027003733317057292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,10240,0.043297068277994795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,2560,0.021771732966105142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,6144,0.02847359975179036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,2560,0.024565333127975465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,2048,0.017944532632827758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,2048,0.022677334149678548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,7168,0.03215466737747193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,1536,0.014200533429781595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,1536,0.020568533738454183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,8192,0.03615573247273763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,1024,0.01027733286221822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,1024,0.019080533583958944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,3072,0.016733866930007935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,768,0.008425600330034892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,768,0.017552000284194947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,4096,0.02093440095583598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,512,0.006670933465162914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,512,0.015552000204722086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,5120,0.024626133839289348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,256,0.003869866579771042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,256,0.015210666259129844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,3584,0.018847999970118205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,128,0.003606399893760681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,128,0.015009066462516785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,1536,0.011064533392588298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,64,0.0033141332368055976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,64,0.014929067095120749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,32,0.0034314667185147605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,32,0.015069866180419922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,2560,0.01463573376337687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,65536,0.3860362688700358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,65536,0.2180799961090088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,2048,0.012478933731714884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,1024,0.009170132875442504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,51200,0.3065962791442871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,51200,0.17721813519795734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,16384,0.10618240038553875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,16384,0.06571199893951415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,12288,0.07189973195393881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,256,0.005224533379077911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,12288,0.05318933327992757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,512,0.0056533331672350565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,768,0.00746559997399648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,10240,0.060444800059000645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,128,0.005004799862702688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,10240,0.046876800060272214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,8192,0.04931413332621257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,8192,0.040563201904296874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,7168,0.0436245322227478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,7168,0.0373802661895752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,6144,0.0383573333422343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,6144,0.034526932239532473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,65536,0.19552213350931805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,5120,0.03717546860376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,5120,0.030998400847117108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,51200,0.1530176003774007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,4096,0.02723199923833211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,16384,0.053641601403554284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,4096,0.02737706700960795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,3584,0.023924267292022704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,3584,0.02566293279329936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,12288,0.04238400061925252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,3072,0.020947200059890748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,3072,0.024392533302307128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,10240,0.035947732130686444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,2560,0.018158932526906334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,2560,0.022510933876037597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,7168,0.027075199286142985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,2048,0.015030399958292643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,2048,0.021332265933354695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,6144,0.023938133319218954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,1536,0.01181653340657552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,1536,0.019836799303690592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,8192,0.030244266986846922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,1024,0.008808533350626629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,1024,0.01771199901898702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,5120,0.020856533447901407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,768,0.007470933099587758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,768,0.01564906636873881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,4096,0.017720532417297364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,512,0.005967999994754791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,512,0.015923200050989787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,3584,0.016220800081888833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,256,0.00378560001651446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,256,0.015384533007939658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,3072,0.0143477330605189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,128,0.0035189333061377203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,128,0.014863999684651694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,2048,0.011383466919263204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,64,0.003324799984693527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,64,0.015262933572133383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,32,0.003432533393303553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,32,0.015305599570274353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,65536,0.3521962801615397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,1024,0.008109866579373678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,2560,0.013237333297729493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,65536,0.20658772786458335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,51200,0.15512107213338217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,1536,0.009875200192133586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,16384,0.09005119800567626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,512,0.005624533196290334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,16384,0.05979733467102051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,51200,0.27365334828694665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,12288,0.06992426713307699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,768,0.006071466704209646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,12288,0.053445335229237875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,10240,0.05358506838480631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,10240,0.043295999368031815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,8192,0.04349013169606526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,8192,0.03803093433380127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,256,0.005294933418432872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,7168,0.039288532733917234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,7168,0.03556693394978841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,128,0.0052373334765434265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,6144,0.034397868315378825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,6144,0.03253333369890849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,65536,0.17763733863830566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,5120,0.033829331398010254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,5120,0.029198932647705077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,12288,0.03868800004323324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,4096,0.02401813268661499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,51200,0.1411861260732015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,4096,0.025990400711695356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,3584,0.02135573426882426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,3584,0.024291199445724488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,16384,0.04951680103937785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,3072,0.018920532862345376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,3072,0.02336853345235189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,10240,0.03325120011965434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,2560,0.01614400049050649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,2560,0.02187626759211222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,6144,0.021967999140421548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,2048,0.013215999801953635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,8192,0.02775786717732747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,2048,0.020942932367324828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,7168,0.02500373323758443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,1536,0.010955733060836793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,1536,0.01940586765607198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,1024,0.008303999900817871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,1024,0.016591999928156534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,3072,0.013174399733543396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,768,0.0070783997575442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,4096,0.017987199624379478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,5120,0.019258666038513183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,512,0.0043605332573254905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,512,0.015867732961972556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,2560,0.012070399522781373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,256,0.003701333453257879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,3584,0.014625066518783569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,256,0.015202132860819497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,128,0.003419733295838038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,128,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,768,0.015740799903869628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,64,0.0032586666444937387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,64,0.015179733435312906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,32,0.0032746667663256326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,32,0.015256533026695251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,2048,0.010475732882817586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,1536,0.009220266342163086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,65536,0.18242026964823405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,65536,0.2910037358601888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,51200,0.14605013529459637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,51200,0.2309877395629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,16384,0.08498240311940511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,16384,0.05459733406702677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,1024,0.006796800096829732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,12288,0.060393599669138584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,12288,0.04465920130411784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,128,0.004858666658401489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,256,0.005106133222579956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,10240,0.05130773385365804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,512,0.0053951998551686605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,10240,0.03925120035807292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,8192,0.03957120180130005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,768,0.005719466507434845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,8192,0.0353653351465861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,7168,0.03400213321050008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,7168,0.03215893308321635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,6144,0.03096533417701721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,6144,0.029605333010355634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,65536,0.1573184013366699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,5120,0.028860799471537274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,5120,0.02675519982973735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,16384,0.044093867142995194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,4096,0.021172267198562623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,4096,0.024578134218851723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,51200,0.1255616029103597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,3584,0.021294933557510377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,3584,0.02324159940083822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,12288,0.03461440006891887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,3072,0.016778665781021117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,3072,0.02245866656303406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,6144,0.019876267512639365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,2560,0.014847999811172486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,2560,0.02103466590245565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,10240,0.029918932914733888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,7168,0.02249280015627543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,2048,0.011929600437482198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,2048,0.020194133122762047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,1536,0.010194133718808491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,1536,0.019006933768590292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,8192,0.024754132827123007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,1024,0.007587199906508128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,1024,0.015629866719245912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,4096,0.014570666352907815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,768,0.006632533172766368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,5120,0.017335466543833413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,768,0.015335466464360556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,512,0.0041407999893029535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,512,0.015467733144760132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,3584,0.01360319952170054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,256,0.003565866748491923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,256,0.015228799978892007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,3072,0.012122666835784912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,128,0.003256533294916153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,128,0.015030399958292643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,1536,0.00869866708914439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,64,0.00325546662012736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,64,0.014893866578737893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,32,0.0033418667813142145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,2560,0.011078400413195293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,32,0.014854400356610616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,65536,0.14612479209899903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,65536,0.2507232030232748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,2048,0.009718400239944459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,1024,0.0059445331494013464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,51200,0.19955946604410807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,51200,0.12099520365397136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,16384,0.05066879987716675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,16384,0.07298133373260499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,12288,0.050969600677490234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,12288,0.03982400099436442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,256,0.004911999901135763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,512,0.005176533261934916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,128,0.004675200084845225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,10240,0.043160533905029295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,8192,0.03611413240432739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,8192,0.03203199903170268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,768,0.005510400235652924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,10240,0.03597226540247599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,7168,0.028905600309371948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,7168,0.029423999786376952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,12288,0.03097599943478902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,6144,0.025597866376241046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,6144,0.026858667532602947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,65536,0.13882880210876464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,5120,0.021924267212549843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,5120,0.024762666225433348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,4096,0.018272000551223754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,4096,0.022808533906936646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,51200,0.11141866842905682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,3584,0.01627840002377828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,3584,0.021983999013900756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,16384,0.039049601554870604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,3072,0.014383999506632486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,3072,0.021154133478800456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,10240,0.02684906721115112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,2560,0.012335999806722005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,2560,0.020437333981196085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,8192,0.02222506602605184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,2048,0.010362666845321656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,2048,0.019422932465871175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,7168,0.02019946575164795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,1536,0.008744532863299053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,1536,0.01747200091679891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,6144,0.01772159934043884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,1024,0.007001600166161854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,1024,0.015562666455904641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,4096,0.013335466384887695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,768,0.005704533557097117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,768,0.01607039968172709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,5120,0.015533866484959922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,512,0.003961600114901861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,3072,0.011282133062680562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,512,0.01578133304913839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,256,0.0035135999321937563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,256,0.015042133132616677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,3584,0.012422399719556172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,128,0.003172266731659571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,128,0.01495253344376882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,2560,0.010496000448862713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,64,0.0030250666042168934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,64,0.015177599589029946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,32,0.003065599997838338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,32,0.015079466501871744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,65536,0.19557120005289713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,2048,0.009136000275611877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,65536,0.12690880298614501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,1536,0.0075552001595497135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,51200,0.15468586285909017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,51200,0.10349439779917399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,16384,0.05691946744918823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,16384,0.044529068470001223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,1024,0.005834666887919108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,12288,0.041145598888397215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,12288,0.03583466609319051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,256,0.005006933212280273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,128,0.0046517332394917805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,10240,0.03544106483459473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,10240,0.032257066170374556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,8192,0.029649066925048827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,768,0.005434666574001312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,8192,0.028945066531499225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,512,0.005098666747411092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,7168,0.026826665798823042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,7168,0.026639999945958455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,6144,0.021498666206995646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,6144,0.025165865818659466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,65536,0.11424427032470703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,5120,0.020768000682195028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,5120,0.023333332935969033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,51200,0.09022933642069499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,4096,0.017237333456675212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,4096,0.022005333503087362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,16384,0.03258240024248759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,3584,0.013455999890963235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,3584,0.020897066593170165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,12288,0.026012800137201947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,3072,0.012245333194732666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,3072,0.019987199703852335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,10240,0.022392533222834268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,2560,0.010541866223017376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,8192,0.0186463991800944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,2560,0.019163733720779418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,2048,0.009164800246556599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,7168,0.01683733264605204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,6144,0.015026133259137472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,1536,0.00759680022795995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,1536,0.016204800208409628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,2048,0.018155733744303383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,1024,0.006177066763242086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,1024,0.01544533371925354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,5120,0.013387733697891235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,768,0.004328533510367076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,768,0.01548479994138082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,4096,0.011760000387827556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,512,0.003717333326737086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,512,0.015209600329399109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,3584,0.01112320025761922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,3072,0.010201600193977357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,256,0.0032768001159032187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,256,0.014870400230089823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,2048,0.008343467116355896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,128,0.003017599880695343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,128,0.014604799946149192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,2560,0.009499733646710713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,64,0.002998399982849757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,1536,0.0065311998128890995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,64,0.014622933665911355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,32,0.0028864001234372456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,32,0.014673067132631936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,65536,0.16919679641723634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,1024,0.005681066711743673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,51200,0.13383785883585614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,51200,0.09173546632130941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,16384,0.04968746503194173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,65536,0.11321492989857991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,16384,0.042345599333445234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,12288,0.04047573407491048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,12288,0.03540906508763631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,768,0.005415466427803039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,10240,0.032415999968846636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,512,0.005111466844876607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,10240,0.030290132761001586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,8192,0.028146133820215864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,8192,0.02648319999376933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,256,0.004836266736189524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,128,0.004633600016434988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,7168,0.02478933334350586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,7168,0.024906667073567708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,6144,0.022382932901382446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,6144,0.023774933815002442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,65536,0.10544959704081218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,5120,0.01656426688035329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,5120,0.022459733486175536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,51200,0.08389653364817301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,4096,0.013899733622868856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,4096,0.021780266364415487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,16384,0.030089600880940752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,3584,0.0123989333709081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,3584,0.020439465840657554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,12288,0.024270933866500855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,3072,0.010995200276374817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,3072,0.01951040029525757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,10240,0.02086506684621175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,2560,0.00986240009466807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,8192,0.017483733097712197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,2560,0.018969599405924478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,2048,0.00899839997291565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,2048,0.016780799627304076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,7168,0.015466666221618653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,1536,0.0072522665063540145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,1536,0.016160000363985697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,6144,0.014133333166440328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,1024,0.0053951998551686605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,1024,0.015638400117556253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,5120,0.012678399682044983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,768,0.004046933352947235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,768,0.016062933206558227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,4096,0.01118293305238088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,512,0.0037920000652472176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,512,0.01516480048497518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,3072,0.009761066238085429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,256,0.0032853332658608755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,256,0.014849066734313965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,2048,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,128,0.0030389333764712016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,3584,0.010690133770306904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,128,0.014760532975196838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,64,0.00290133332212766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,64,0.01487573285897573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,32,0.0029653333127498626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,32,0.01492693324883779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,65536,0.14729493459065754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,2560,0.009019733468691508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,65536,0.10101119677225749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,51200,0.11704533100128174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,51200,0.08300373554229737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,1024,0.005669333537419637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,16384,0.04171946843465169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,16384,0.03738986651102702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,1536,0.006516266862551372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,12288,0.032953600088755294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,12288,0.03132159908612569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,768,0.005275733272234599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,512,0.005129600067933401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,10240,0.028461867570877077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,10240,0.02834986646970113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,8192,0.02345386743545532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,8192,0.02518293261528015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,256,0.0048767998814582825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,7168,0.02104426622390747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,7168,0.023820799589157105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,128,0.004732800026734671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,6144,0.01729493339856466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,6144,0.022682666778564453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,65536,0.10714773337046306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,5120,0.014717866977055868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,5120,0.021625600258509316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,51200,0.08606186707814535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,16384,0.03020906647046407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,4096,0.012222933769226074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,12288,0.024027733008066814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,4096,0.020657066504160562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,3584,0.011225600043932598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,3584,0.02022506594657898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,10240,0.020750933885574342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,3072,0.010238933563232421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,3072,0.019552000363667808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,8192,0.017432532707850137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,2560,0.00909546713034312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,2560,0.018218666315078735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,6144,0.013906133174896241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,2048,0.008202666540940602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,2048,0.016174933314323424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,1536,0.006958933174610138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,1536,0.01597760021686554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,7168,0.015587199727694193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,1024,0.0047082667549451195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,1024,0.015800533692042033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,4096,0.01112320025761922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,768,0.004046933352947235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,5120,0.012731732924779257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,768,0.015636266271273295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,512,0.0036533333361148832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,512,0.015063466628392539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,3072,0.009457066655158997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,256,0.003319466610749563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,256,0.014913066228230795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,1536,0.006185600161552429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,128,0.002996266633272171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,128,0.014647466937700906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,3584,0.010372266173362732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,64,0.0029706666866938275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,64,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,2560,0.008500267068545024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,32,0.002977066735426585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,32,0.014703999956448874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,65536,0.12235306898752848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,65536,0.08841813405354818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,2048,0.007110400001207988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,51200,0.09697706699371338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,51200,0.07223359743754068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,16384,0.035327998797098796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,16384,0.034790400664011636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,1024,0.005667200187842051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,12288,0.028099199136098225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,12288,0.02898026704788208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,768,0.005302399893601735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,512,0.004970666766166687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,10240,0.024647466341654458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,10240,0.02612053354581197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,8192,0.02062186598777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,8192,0.02399253249168396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,256,0.0047658666968345646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,7168,0.01722773313522339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,7168,0.023113600413004556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,128,0.00469760000705719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,6144,0.015187199910481772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,6144,0.02188373406728109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,65536,0.10078399976094563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,5120,0.01316159963607788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,5120,0.021172267198562623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,51200,0.08104746341705323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,16384,0.028284800052642823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,4096,0.011017599701881408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,4096,0.019565866390864054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,3584,0.010197333494822184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,12288,0.022562134265899658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,3584,0.018949333826700845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,3072,0.009351467092831928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,3072,0.01737706661224365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,10240,0.019591466585795084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,2560,0.008414933085441589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,2560,0.016720000902811685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,8192,0.016522666811943053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,2048,0.007344000041484833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,7168,0.014900267124176025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,6144,0.01359999974568685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,1536,0.006248533229033152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,1536,0.016312533617019655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,2048,0.01678933302561442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,1024,0.004402133325735727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,1024,0.016189866264661155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,3072,0.008450133601824443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,768,0.004008533308903376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,768,0.015316266814867655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,3584,0.010145066181818645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,512,0.003603200117746989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,5120,0.012177067001660664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,4096,0.010732799768447876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,512,0.015126400192578635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,256,0.014965333541234336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,256,0.003337600082159042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,128,0.0030826665461063385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,128,0.014628266294797262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,1536,0.0061482667922973635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,64,0.0029002666473388673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,2560,0.007938133180141449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,64,0.014753066500027976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,32,0.002932266642649968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,32,0.014812800288200378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,65536,0.10366293589274089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,2048,0.007012266914049785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,65536,0.07665279706319174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,51200,0.07950507005055746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,51200,0.0636351982752482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,1024,0.005712000032265982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,16384,0.029441066582997638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,16384,0.03095146616299947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,768,0.0052373334765434265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,256,0.004834133386611939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,12288,0.02369813323020935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,12288,0.02685333291689555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,512,0.005050666630268097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,10240,0.020761599143346153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,10240,0.02462293306986491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,8192,0.017018665870030723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,8192,0.02253546714782715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,7168,0.01539520025253296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,7168,0.021699200073877968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,128,0.004737066725889841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,6144,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,6144,0.02152959903081258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,65536,0.08336319923400878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,5120,0.012191999951998394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,5120,0.020433066288630168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,51200,0.06684587001800538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,4096,0.01064639985561371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,4096,0.019682133197784425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,16384,0.023853866259257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,3584,0.009371733665466309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,3584,0.01878933310508728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,10240,0.01672853430112203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,3072,0.008501332998275758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,12288,0.01918826699256897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,3072,0.017500799894332886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,2560,0.007663999994595845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,8192,0.01453013320763906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,2560,0.01686720053354899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,2048,0.006877866884072621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,2048,0.016749866803487144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,7168,0.013196800152460733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,1536,0.00527999997138977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,1536,0.01602240006128947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,6144,0.011947733163833619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,1024,0.00421973317861557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,1024,0.015784533818562825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,5120,0.010987733801205952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,768,0.0039018665750821433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,768,0.015447466572125753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,4096,0.009731200337409974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,512,0.0036117332677046456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,512,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,3584,0.009019733468691508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,256,0.0032479998966058097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,2048,0.0065760001540184024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,3072,0.007891199986139933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,256,0.014834133783976236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,128,0.003010133405526479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,128,0.014644267161687216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,64,0.002903466671705246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,64,0.014619732896486918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,32,0.0029535998900731405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,32,0.014738133549690247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,1024,0.005404800176620483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,2560,0.007367466886838277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,65536,0.07919253508249918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,65536,0.06614826520284017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,51200,0.06290453275044759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,51200,0.05444586674372355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,1536,0.006119466821352641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,16384,0.023947733640670776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,16384,0.027210666735966997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,12288,0.019142399231592812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,12288,0.024247467517852783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,768,0.005188266436258951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,10240,0.01649386684099833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,10240,0.0228383998076121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,512,0.005006933212280273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,8192,0.01362773378690084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,8192,0.02179093360900879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,256,0.004717866579691568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,128,0.004614399870236715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,7168,0.012798933188120523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,7168,0.02177919944127401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,6144,0.011482666929562886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,6144,0.020921599864959717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,65536,0.08081066608428955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,5120,0.010542933146158855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,16384,0.023264000813166298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,5120,0.019594667355219524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,4096,0.009517866373062133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,4096,0.017835734287897746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,12288,0.01881813406944275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,3584,0.008628267049789428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,3584,0.01773653427759806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,10240,0.01655359963575999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,3072,0.007963733375072479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,8192,0.014284800489743552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,3072,0.017323732376098633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,51200,0.06425919930140177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,2560,0.007143466671307881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,2560,0.016724266608556113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,7168,0.012989866733551025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,2048,0.005907199780146281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,2048,0.016499200463294984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,6144,0.011846400300661723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,1536,0.004889599978923798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,1536,0.01609280010064443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,1024,0.004188799858093261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,1024,0.015397333105405173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,5120,0.010706133643786113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,768,0.003806933263937632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,768,0.01535040040810903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,4096,0.008952533205350239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,512,0.0035616000493367515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,512,0.015056000153223673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,3584,0.00842133363087972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,3072,0.007693866888682048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,256,0.0031541332602500914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,256,0.014838400483131408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,128,0.0030069333811601003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,128,0.014578133821487427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,2560,0.007241599758466085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,64,0.0028480000793933867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,64,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,32,0.002865066627661387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,32,0.01458560029665629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,65536,0.05414613485336304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,2048,0.006460799773534138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,65536,0.05321280161539713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,51200,0.043483734130859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,51200,0.045468799273173016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,1024,0.005399466554323832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,16384,0.017106133699417114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,1536,0.006071466704209646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,16384,0.023569067319234215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,12288,0.013704533378283182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,12288,0.022451200087865195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,768,0.0050805335243542995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,10240,0.012345600128173827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,10240,0.021501866976420085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,512,0.004863999783992767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,8192,0.010583466291427613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,8192,0.020296533902486168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,256,0.004760533571243286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,128,0.004666666686534882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,7168,0.009913600484530131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,7168,0.019826134045918785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,6144,0.009998933474222819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,6144,0.018964266777038573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,65536,0.07977920373280843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,5120,0.009173333644866943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,5120,0.01918506622314453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,51200,0.06344106594721476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,4096,0.008248533308506011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,4096,0.018133334318796792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,12288,0.018572799364725747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,3584,0.007418666779994964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,16384,0.02300693392753601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,3584,0.01770240068435669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,3072,0.006852266689141591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,3072,0.01709973414738973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,10240,0.016317866245905557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,2560,0.006239999830722809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,2560,0.01641706625620524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,8192,0.01406826674938202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,2048,0.0055167997876803074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,2048,0.016265599926312765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,7168,0.012686933080355326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,1536,0.004902400076389313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,1536,0.016178133090337117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,6144,0.0113045334815979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,1024,0.004222933451334635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,1024,0.015691733360290526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,5120,0.009830400347709656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,768,0.0038431999584039056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,768,0.01527466674645742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,4096,0.008785067001978557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,512,0.0034954667091369627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,512,0.014816000064214071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,3584,0.008252800007661184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,256,0.0031456001102924346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,256,0.014781866470972696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,3072,0.007648000121116638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,128,0.0029525332152843474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,128,0.014605866869290671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,2048,0.0064650664726893115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,2560,0.007147733370463054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,64,0.002867199977238973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,32,0.002948266764481862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,64,0.014662399888038635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,32,0.014599466323852539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,65536,0.043030401070912674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,65536,0.0479477326075236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,51200,0.03510826826095581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,51200,0.03976426521937053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,1536,0.006102400024731954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,1024,0.005236266553401947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,16384,0.013858133554458618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,16384,0.02285226583480835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,768,0.0053258667389551794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,12288,0.011611732840538024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,12288,0.02140799959500631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,512,0.004863999783992767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,10240,0.01104213297367096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,10240,0.020177066326141357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,256,0.0046965335806210835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,8192,0.009698133667310078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,8192,0.01921280026435852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,7168,0.009149866302808125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,7168,0.019363200664520262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,128,0.004508799811204275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,6144,0.008483200271924337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,6144,0.018235733111699425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,65536,0.07897919813791911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,5120,0.0078015998005867004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,5120,0.01830613414446513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,51200,0.0632863998413086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,4096,0.006682666639486949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,4096,0.018075732390085857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,16384,0.02295680046081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,3584,0.00692799985408783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,3584,0.017630932728449504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,12288,0.018540799617767334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,3072,0.006355200211207073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,3072,0.017102932929992674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,10240,0.016389333208402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,2560,0.0062047998110453285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,2560,0.016540799538294473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,8192,0.013579733173052468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,2048,0.005508266886075338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,2048,0.016226133704185484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,7168,0.012157866358757019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,1536,0.0047765334447224935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,1536,0.01601599951585134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,6144,0.01074773371219635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,1024,0.0040448000033696495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,1024,0.01620586713155111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,5120,0.009788800279299419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,768,0.0037503999968369803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,768,0.01515733301639557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,4096,0.008648533622423809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,512,0.003533866753180822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,512,0.015006933609644571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,3072,0.007560533285140991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,256,0.003176533430814743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,256,0.014670933286348978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,2560,0.007161599894364675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,128,0.0028789333999156954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,128,0.014426666498184203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,3584,0.008280533552169799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,2048,0.006450133522351582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,64,0.002794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,32,0.002867199977238973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,64,0.014719999829928079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,65536,0.031531733274459836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,32,0.014321066935857139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,65536,0.04054506619771321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,51200,0.025886933008829754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,1024,0.005238399902979533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,51200,0.033931732177734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,16384,0.010613333185513813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,16384,0.020874667167663574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,1536,0.005963733295599619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,768,0.004983466863632202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,12288,0.009788800279299419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,12288,0.019166932503382365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,10240,0.008890666564305623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,10240,0.01919680039087931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,512,0.004866133133570353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,8192,0.008026666442553202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,8192,0.018683733542760213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,256,0.004747733473777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,7168,0.007236266632874806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,7168,0.018812799453735353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,128,0.004448000093301137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,6144,0.0066549330949783325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,6144,0.018285866578420004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,65536,0.07943466504414877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,5120,0.0066655998428662615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,5120,0.018660267194112144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,51200,0.0626698652903239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,4096,0.006209066510200501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,4096,0.01819733381271362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,16384,0.02294506629308065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,3584,0.006681600213050842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,3584,0.017627733945846557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,10240,0.015485866864522298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,3072,0.00626986672480901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,3072,0.017095466454823814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,12288,0.017860267559687296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,2560,0.006083199878533682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,2560,0.01672640045483907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,8192,0.013191466530164083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,2048,0.005449600021044413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,2048,0.01652479966481527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,7168,0.01186133325099945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,6144,0.01079360047976176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,1536,0.004675200084845225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,1536,0.01592853367328644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,5120,0.00974720021088918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,1024,0.004088533421357473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,1024,0.015477333466211954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,4096,0.008584533135096233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,768,0.0037418665985266366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,768,0.01541866660118103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,512,0.00347626656293869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,3072,0.007582933207352956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,512,0.015027200182278952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,256,0.0030858665704727173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,3584,0.008280533552169799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,256,0.014849066734313965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,128,0.002926933268706004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,128,0.014498133460680643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,2048,0.006305066744486491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,64,0.002696533252795537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,64,0.014638933539390563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,32,0.002699733277161916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,32,0.014713600277900696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,65536,0.01917333404223124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,65536,0.03167146643002828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,2560,0.007145600020885467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,51200,0.014994133512179056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,51200,0.027486934264500933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,1536,0.005981866518656413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,16384,0.008065066734949748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,16384,0.02015786568323771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,768,0.005110399921735128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,12288,0.0069248000780741375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,1024,0.005377066632111868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,12288,0.018905599912007652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,10240,0.006664533416430156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,10240,0.019008000691731773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,512,0.004851200183232625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,128,0.004498133560021719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,8192,0.006392533580462138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,256,0.004680533210436503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,7168,0.006340266764163971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,8192,0.0184063990910848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,7168,0.01889386574427287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,6144,0.006159999966621399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,6144,0.018396800756454466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,65536,0.07872746785481771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,5120,0.006468266745408376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,5120,0.0189301331837972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,51200,0.062495998541514074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,4096,0.006095999975999197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,4096,0.017781333128611247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,16384,0.02193173368771871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,3584,0.006538666784763336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,3584,0.01728960076967875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,12288,0.01771413286526998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,3072,0.006192000210285186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,10240,0.015288533767064414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,3072,0.01688213348388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,2560,0.006046933432420095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,2560,0.016557866334915163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,8192,0.013140267133712769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,2048,0.005339733262856802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,2048,0.016178133090337117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,7168,0.0118559996287028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,1536,0.004763733347256978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,6144,0.010931199789047242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,1536,0.016151466965675355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,1024,0.004105599969625473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,1024,0.015318399667739869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,5120,0.009717333316802978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,768,0.003738666574160258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,768,0.015412267049153647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,4096,0.008770133058230083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,512,0.003398400048414866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,512,0.01495039959748586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,3072,0.007605333129564922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,256,0.003094399968783061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,256,0.014539733529090881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,2048,0.006400000055631001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,128,0.0028618666032950084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,128,0.014416000247001648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,1024,0.0053375999132792154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,64,0.0027242665489514667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,64,0.014507733782132468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,32,0.002739199995994568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,2560,0.007186133166154225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,32,0.014590932925542196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,65536,0.012289067109425861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,65536,0.028355199098587035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,51200,0.010675199826558431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,51200,0.02518080075581868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,1536,0.005995733539263407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,16384,0.006265600025653839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,16384,0.01958613395690918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,3584,0.008178133269151051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,12288,0.006138666470845541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,12288,0.0187008003393809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,768,0.0050901333491007485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,10240,0.006397866706053417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,10240,0.01885546644528707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,512,0.0048981333772341405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,256,0.004603733122348785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,8192,0.006281599899133046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,128,0.004524800181388855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,8192,0.01802026629447937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,7168,0.006082133452097575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,7168,0.018684800465901694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,6144,0.005980800092220307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,65536,0.0781599998474121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,6144,0.018332799275716148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,5120,0.006333866715431213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,5120,0.018692266941070557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,51200,0.061860267321268717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,4096,0.005894400179386139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,4096,0.01837653319040934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,16384,0.02209386626879374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,12288,0.01771519978841146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,3584,0.006435200075308482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,3584,0.016923733552296958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,3072,0.006054399907588959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,3072,0.016838399569193523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,10240,0.015554133057594299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,2560,0.006074666480223338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,8192,0.013142399986584983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,2560,0.01704746683438619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,2048,0.0053269331653912864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,2048,0.016404267152150473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,7168,0.011868799726168316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,1536,0.004700799783070883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,1536,0.015474133690198264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,6144,0.010857599973678588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,1024,0.0040224000811576845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,1024,0.01552959978580475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,5120,0.009752532839775086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,768,0.0037151999771595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,768,0.0151637335618337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,4096,0.008584533135096233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,3072,0.007632000247637431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,512,0.0150218665599823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,3584,0.008389332890510559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,256,0.0030794667700926462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,256,0.014747732877731323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,512,0.00342399999499321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,128,0.0028959999481836954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,128,0.01443839967250824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,2048,0.006343466540177663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,64,0.0026975999275843303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,2560,0.0071168000499407455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,32,0.0027232001225153605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,1536,0.005937066674232483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,65536,0.009166933099428813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,32,0.014389333128929139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,51200,0.008311466872692108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,65536,0.02686079939206441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,16384,0.006080000102519989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,51200,0.024360533555348715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,12288,0.0062047998110453285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,16384,0.018988800048828126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,10240,0.006217599908510844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,12288,0.019141334295272826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,8192,0.0061471998691558834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,8192,0.017844265699386595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,7168,0.005994666616121928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,7168,0.018606932957967122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,1024,0.005260799825191498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,6144,0.005864533285299936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,5120,0.006217599908510844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,6144,0.0180949330329895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,4096,0.0059114664793014525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,768,0.005160533388455709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,5120,0.018501333395640054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,3584,0.006235733131567637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,4096,0.017656532923380534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,3584,0.017144532998402913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,3072,0.00595413347085317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,2560,0.006033066908518473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,3072,0.0168938676516215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,512,0.004853333532810211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,2048,0.0054058666030565895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,2560,0.016463999946912132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,1536,0.0046410664916038515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,2048,0.016394666830698647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,1024,0.004053333401679992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,1536,0.016267733772595723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,768,0.0036447999378045404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,512,0.003416533271471659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,768,0.015396266182263692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,256,0.0030080000559488933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,512,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,128,0.0027935999135176343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,256,0.014755200346310934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,64,0.0026602665583292644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,128,0.014405333002408347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,32,0.0026986666023731233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,64,0.014693333705266317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,32,0.014433067043622336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,65536,0.008381866415341695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,51200,0.009217066566149394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,65536,0.02764799992243449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,16384,0.006241066753864289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,51200,0.023949867486953734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,12288,0.006033066908518473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,16384,0.01958613395690918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,10240,0.006198399762312571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,12288,0.018711467583974205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,256,0.0046965335806210835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,8192,0.006097066899140676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,10240,0.01872746745745341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,10240,0.018550399939219156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,7168,0.005958400170008342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,8192,0.01817493240038554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,6144,0.005743999779224396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,7168,0.018580265839894614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,6144,0.018026665846506754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,5120,0.018685867389043175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,3584,0.006080000102519989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,4096,0.017678932348887125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,3072,0.00580266664425532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,3584,0.017318399747212727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,2560,0.005859200159708659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,3072,0.017449599504470826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,128,0.004557866851488749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,2048,0.005332266787687937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,2560,0.016858667135238647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,1536,0.004621866842110952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,2048,0.016130133469899496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,1024,0.003917866696914038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,1536,0.015477333466211954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,768,0.003571200122435888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,1024,0.01520853340625763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,512,0.003310933212439219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,768,0.014986667037010192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,256,0.0030271999537944795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,512,0.015914666652679443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,256,0.014443733294804893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,64,0.0026122666895389557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,1024,0.015098667144775391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,128,0.01427839994430542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,64,0.014300800363222756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,32,0.002648533384005229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,32,0.014320000012715658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,64,0.014547200004259745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,5120,0.006201600035031637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,4096,0.005734399954477946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,16384,0.769920031229655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,128,0.0028437333802382152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,16384,0.40454721450805664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,12288,0.30997438430786134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,12288,0.5899679819742839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,51200,1.2325546264648437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,10240,0.27030506134033205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,10240,0.4903744061787923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,8192,0.3909248034159342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,8192,0.21486293474833168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,51200,2.433727010091146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,7168,0.34415785471598304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,7168,0.19065920511881512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,6144,0.16484053929646808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,16384,0.36823358535766604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,6144,0.2956586519877116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,5120,0.247869873046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,12288,0.27754666010538737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,51200,1.1292607625325521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,5120,0.14092159271240234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,4096,0.20046079953511559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,4096,0.13178239663441976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,3584,0.17687679926554362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,10240,0.2693984031677246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,3584,0.11949013074239094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,3072,0.09264000256856283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,3072,0.16816426912943522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,8192,0.18923199971516927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,2560,0.12748479843139648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,2560,0.08147306442260742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,2560,0.0655679980913798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,2048,0.10236053466796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,2048,0.06680853366851806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,7168,0.19076372782389323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,1536,0.07985493342081705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,1536,0.06205653349558512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,1024,0.0627349336942037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,1024,0.047040001551310225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,6144,0.14411733945210775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,768,0.04931519826253255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,5120,0.12558293342590332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,768,0.03728640079498291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,4096,0.09988479614257813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,512,0.03044160008430481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,512,0.029661866029103597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,256,0.01758293310801188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,256,0.02225066622098287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,3584,0.08920746644337972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,128,0.011121066411336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,128,0.01951040029525757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,3072,0.07762239774068197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,64,0.007981866598129272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,64,0.017544533809026083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,65536,32,0.00592853327592214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,65536,32,0.01795626680056254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,2048,0.05469333330790201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,1536,0.042565333843231204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,1024,0.03179840048154195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,512,0.020089600483576456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,768,0.028567467133204145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,256,0.014274133245150247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,65536,1.2790847778320313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,65536,128,0.01225920021533966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,65536,2.413565826416016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,16384,0.3339285214742025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,16384,0.6478549321492513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,12288,0.2773098627726237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,51200,1.0146602630615233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,12288,0.45400746663411456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,10240,0.2332767963409424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,10240,0.38605546951293945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,51200,1.86693115234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,8192,0.168994140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,8192,0.30099519093831384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,7168,0.14857813517252605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,65536,1.1324799855550132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,6144,0.26767358779907224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,7168,0.26814613342285154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,6144,0.13030292987823486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,16384,0.3379754702250163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,5120,0.19146025975545247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,12288,0.219978666305542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,51200,0.8816533406575522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,5120,0.12533439795176188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,4096,0.1542922655741374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,4096,0.08742720286051432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,4096,0.10351786613464356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,3584,0.13410879770914713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,3584,0.08384213447570801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,3072,0.1160746653874715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,3584,0.07923733393351237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,3072,0.07397653261820475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,2560,0.09938986301422119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,2560,0.0643285314242045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,8192,0.17192427317301434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,2048,0.08050026893615722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,2048,0.06011626720428467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,1536,0.0696394681930542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,1536,0.049421866734822587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,7168,0.13187626997629803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,10240,0.19441599845886232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,1024,0.04875306685765584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,1024,0.03903146584828694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,768,0.03349440097808838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,768,0.03097493251164754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,6144,0.11420266628265381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,512,0.026605866352717084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,512,0.02498240073521932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,5120,0.0964202642440796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,256,0.013299199938774108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,256,0.020623999834060668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,3072,0.06127253373463949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,128,0.00934933324654897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,128,0.017414400974909462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,2048,0.04366933504740397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,64,0.005603200197219849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,64,0.017318399747212727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,51200,32,0.005624533196290334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,51200,32,0.01736853321393331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,1536,0.03451626698176066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,1024,0.026213334004084272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,768,0.0214026669661204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,2560,0.05332159996032715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,65536,0.38557545344034827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,65536,0.7506111780802409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,51200,0.5908543904622395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,51200,0.3055434544881185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,16384,0.21965546607971193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,512,0.01634239951769511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,16384,0.1267680009206136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,16384,0.0979967991511027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,12288,0.1410101254781087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,12288,0.08795413176218668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,256,0.012210133671760558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,10240,0.11761386394500732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,10240,0.07501653035481771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,8192,0.09576000372568766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,8192,0.06415253480275472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,7168,0.08423039913177491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,7168,0.05807360013326009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,6144,0.07315626939137777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,6144,0.05245440006256104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,51200,128,0.009080533186594646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,5120,0.061516801516215004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,5120,0.0460970679918925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,65536,0.36563094456990564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,4096,0.05830933252970377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,4096,0.04001813332239787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,51200,0.2888127962748209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,3584,0.05149226586023966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,12288,0.07534506320953369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,3584,0.03704213301340739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,3072,0.039025068283081055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,10240,0.0642186681429545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,3584,0.027670399347941084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,3072,0.034179198741912845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,2560,0.03320320049921672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,2560,0.03100373347600301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,7168,0.04721493323644002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,8192,0.05280426740646362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,2048,0.02746773362159729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,2048,0.027451733748118084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,1536,0.021589332818984987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,1536,0.02397759954134623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,6144,0.04145706494649251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,1536,0.01472106675306956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,1024,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,5120,0.03577173153559367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,768,0.011782399813334147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,768,0.01962453325589498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,1024,0.020849066972732543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,512,0.00895360012849172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,512,0.0176256000995636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,4096,0.03006400068600972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,256,0.006109866499900818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,256,0.015710933009783427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,3072,0.023995733261108397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,128,0.003807999938726425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,128,0.015313067038853965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,2048,0.017972266674041747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,2560,0.021540266275405884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,64,0.003621333340803782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,64,0.015530666708946228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,16384,32,0.003770666569471359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,16384,32,0.01564586659272512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,1024,0.011552000045776367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,65536,0.3097749392191569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,65536,0.5641482671101887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,51200,0.4443850517272949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,768,0.01009279986222585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,51200,0.24789226849873863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,16384,0.17521173159281414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,16384,0.09245013395945231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,12288,0.10709760189056397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,12288,0.07263680299123129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,10240,0.08950293064117432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,10240,0.06269973516464233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,512,0.008519466718037922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,8192,0.07809173266092936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,8192,0.054202667872111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,256,0.006284800171852112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,7168,0.06788053512573242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,7168,0.04920639991760254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,16384,128,0.005794133245944977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,6144,0.05868159929911295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,6144,0.04414186477661133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,65536,0.28066558837890626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,5120,0.04966080188751221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,5120,0.039724798997243245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,51200,0.2349653402964274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,16384,0.07510613600413005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,4096,0.04641706546147664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,12288,0.05862400134404501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,4096,0.034808532396952314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,3584,0.03469013373057048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,3584,0.03263360063234965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,3072,0.0313482662041982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,3072,0.030151466528574627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,10240,0.04975359837214152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,2560,0.0269322673479716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,2560,0.027162667115529376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,8192,0.04165120124816894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,2048,0.022154666980107627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,2048,0.02440213362375895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,2048,0.014327466487884521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,1536,0.01735360026359558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,1536,0.02214933236440023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,7168,0.037006934483846024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,1024,0.012653866410255432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,6144,0.032656000057856245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,1024,0.019963733355204263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,768,0.010282666484514872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,768,0.01857173244158427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,768,0.008771199981371562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,512,0.007906133433183034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,512,0.01602026621500651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,5120,0.028259199857711793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,256,0.0040832000474135075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,256,0.015552000204722086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,4096,0.024014933904012045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,128,0.0035904000202814737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,3584,0.021652267376581828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,128,0.01532586713631948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,64,0.0033482665816942847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,64,0.015377066532770791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,12288,32,0.003472000112136205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,3072,0.01922773321469625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,12288,32,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,2560,0.017108267545700072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,65536,0.26907307306925454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,65536,0.4838250796000163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,1536,0.011799466609954835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,65536,0.23728853861490884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,16384,0.11556053161621094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,51200,0.2154688040415446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,51200,0.380185604095459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,16384,0.0748138666152954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,12288,0.0887391964594523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,1024,0.00981119970480601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,12288,0.061203201611836754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,10240,0.07388373215993246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,10240,0.053311999638875326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,512,0.006498133142789205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,8192,0.06018666823705038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,8192,0.04643946488698324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,256,0.005534933507442474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,7168,0.05373973449071249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,7168,0.042455466588338216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,7168,0.032022400697072344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,6144,0.04732799927393595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,6144,0.03865493138631185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,12288,128,0.005256533126036326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,5120,0.03990506728490194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,5120,0.035019731521606444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,51200,0.18748480478922527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,4096,0.03693653345108032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,16384,0.07503893375396728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,4096,0.03059413234392802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,3584,0.030459733804066975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,3584,0.028698666890462236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,3584,0.018696532646814982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,12288,0.050280535221099855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,3072,0.026514132817586262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,2560,0.02204373280207316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,2560,0.024401066700617473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,10240,0.04292159875233968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,2048,0.018210132916768394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,2048,0.022778666019439696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,8192,0.03577599922815959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,1536,0.014229333400726319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,1536,0.021013333400090536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,6144,0.028275199731190997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,1024,0.010573866963386535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,1024,0.019461333751678467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,3072,0.025700267155965167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,768,0.008748799562454224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,5120,0.02449280023574829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,768,0.016986666123072307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,768,0.007814399898052216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,512,0.006911999980608622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,512,0.015655466914176942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,512,0.006061866879463196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,256,0.004019200056791306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,256,0.015228799978892007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,128,0.0035807999471823373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,128,0.014966400464375815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,4096,0.020823466777801513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,64,0.0032799998919169106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,3072,0.016786134243011473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,64,0.015241600076357522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,10240,32,0.0033386667569478357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,10240,32,0.015127467115720114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,2560,0.014826666315396628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,2048,0.012626133362452188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,65536,0.21277012825012206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,65536,0.38853333791097006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,51200,0.17002132733662922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,1536,0.010660266876220703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,51200,0.3089376131693522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,16384,0.09445866743723551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,16384,0.06410666704177856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,12288,0.07212800184885661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,12288,0.05249919891357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,1024,0.009046399593353271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,10240,0.04607679843902588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,10240,0.06091093222300211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,8192,0.049187199274698896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,256,0.005303466816743215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,8192,0.040415998299916586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,7168,0.04359999895095825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,7168,0.03704426685969035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,10240,128,0.0050805335243542995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,6144,0.03810986677805583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,6144,0.033930667241414386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,65536,0.19200000762939454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,5120,0.03728533188501994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,5120,0.031041065851847332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,51200,0.15182719230651856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,16384,0.06111253499984741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,4096,0.030479999383290608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,4096,0.027258666356404622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,3584,0.02379093368848165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,12288,0.042081065972646076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,3584,0.025782400369644166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,3584,0.016391467054684958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,3072,0.02102933327356974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,3072,0.014659200112024942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,3072,0.023913600047429404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,2560,0.01803306738535563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,8192,0.03024959961573283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,2560,0.022567466894785563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,2048,0.014958932995796204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,2048,0.02113599975903829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,2048,0.011447466413180033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,1536,0.011764267086982727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,1536,0.01954879959424337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,7168,0.02696533401807149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,1024,0.008823466300964356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,1024,0.01832533280054728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,6144,0.02397973338762919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,768,0.0075530668099721265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,768,0.01586666703224182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,10240,0.03607573509216309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,5120,0.020899200439453126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,512,0.006030933558940887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,512,0.01569066643714905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,256,0.0038122666378815973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,4096,0.017934934298197428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,256,0.015158399939537048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,128,0.0034805332620938623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,128,0.015076266725858054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,2560,0.013204266627629599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,64,0.0033642667035261786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,1536,0.00993066628774007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,64,0.014907733599344889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,8192,32,0.00335359995563825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,8192,32,0.01513706644376119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,1024,0.008257066706816356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,768,0.006931200126806895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,65536,0.2001194636027018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,65536,0.3463850657145182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,51200,0.2669301350911458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,16384,0.1022058645884196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,51200,0.13946560223897297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,51200,0.15150933265686034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,12288,0.06373226642608643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,12288,0.04803626537322998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,10240,0.053766401608785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,10240,0.042881067593892416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,16384,0.06290880044301352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,8192,0.04320533275604248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,256,0.005409066875775656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,8192,0.037656533718109134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,128,0.005291733145713806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,7168,0.038585599263509116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,7168,0.03440426588058472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,6144,0.03391893307367961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,6144,0.0316213329633077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,8192,512,0.0060479998588562015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,65536,0.17543892860412597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,5120,0.029165865977605183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,5120,0.029079467058181763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,4096,0.023843199014663696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,16384,0.049157333374023435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,4096,0.02590293288230896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,3584,0.021383466323216756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,12288,0.038599467277526854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,3584,0.024358399709065757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,3072,0.0189194659392039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,3072,0.02305493354797363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,10240,0.03319786588350932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,2560,0.016217600305875143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,2560,0.0217141330242157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,8192,0.027741867303848266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,2048,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,2048,0.020336000124613444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,7168,0.0247381329536438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,1536,0.010833066701889039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,1536,0.019427200158437095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,6144,0.022050132354100548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,1024,0.008142933249473572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,1024,0.01653333306312561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,5120,0.019413334131240845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,768,0.006896000107129414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,768,0.01563093364238739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,4096,0.01643946667512258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,3584,0.01470080018043518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,512,0.00448639988899231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,512,0.015404799580574035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,256,0.003606399893760681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,3072,0.0131221334139506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,256,0.015170133113861084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,128,0.003328000009059906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,128,0.014844800035158793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,2560,0.01222933332125346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,64,0.0031498665610949195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,64,0.015064533551534018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,7168,32,0.0032127998769283296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,7168,32,0.01513920029004415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,65536,0.2886101404825846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,65536,0.1709237257639567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,2048,0.010843732953071594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,1536,0.009180800120035807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,51200,0.23113493919372557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,1024,0.007565866907437642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,51200,0.1380287965138753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,16384,0.07659839789072673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,16384,0.05569493373235067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,12288,0.06267093420028687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,12288,0.04541120131810506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,768,0.006550399959087372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,10240,0.05305279890696207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,10240,0.03977280060450236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,10240,0.02996266682942708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,512,0.005645866692066193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,8192,0.04082560141881307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,8192,0.035461334387461345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,256,0.005184000233809153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,7168,0.03574719826380412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,7168,0.032509867350260416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,7168,128,0.004985600213209788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,6144,0.030502400795618695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,6144,0.02961066762606303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,65536,0.15511892636617025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,5120,0.029819732904434203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,5120,0.027512532472610474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,16384,0.04427306652069092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,51200,0.12416426340738933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,4096,0.022163200378417968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,4096,0.02572373350461324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,3584,0.01989439924558004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,3584,0.024010666211446128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,12288,0.034509865442911784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,3072,0.017288533846537273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,3072,0.02250666618347168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,8192,0.02484053373336792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,2560,0.014885333180427552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,2560,0.02122559944788615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,2048,0.012599466244379678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,7168,0.022510933876037597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,2048,0.019661867618560792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,1536,0.010361599922180175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,6144,0.019887999693552653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,1536,0.008648533622423809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,1536,0.018526933590571084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,1024,0.007595733304818471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,1024,0.015890133380889893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,768,0.006594133377075195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,768,0.01583573321501414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,5120,0.017486933867136636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,512,0.00420053352912267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,512,0.015331199765205384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,512,0.005648000041643778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,256,0.0035818666219711304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,4096,0.015041066209475198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,256,0.015015467007954916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,128,0.0033258666594823206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,128,0.014794666568438211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,3584,0.0134442667166392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,64,0.0031413334111372627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,64,0.014748799800872802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,6144,32,0.0031776001056035364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,6144,32,0.014834133783976236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,65536,0.25337492624918617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,65536,0.15165866216023763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,2560,0.011618133385976155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,2048,0.010115200281143188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,1024,0.006878933310508728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,51200,0.1830560048421224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,51200,0.11657066345214843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,16384,0.06637333234151205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,768,0.006387199958165486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,16384,0.050545068581899014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,16384,0.04008320172627767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,12288,0.051888000965118405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,3072,0.01225600043932597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,12288,0.03950933218002319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,10240,0.04470613400141398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,8192,0.036433064937591554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,8192,0.03160853385925293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,256,0.005053866902987162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,7168,0.02882986664772034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,7168,0.029090134302775066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,6144,128,0.00487253318230311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,6144,0.02566933234532674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,6144,0.026884265740712482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,65536,0.14112639427185059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,5120,0.02406186660130819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,51200,0.11049386660257976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,5120,0.025035732984542848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,12288,0.031380265951156616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,4096,0.01818559964497884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,4096,0.02288533250490824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,3584,0.01644266645113627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,10240,0.02732693354288737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,3584,0.022055466969807945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,3072,0.014459733168284097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,3072,0.02102186679840088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,8192,0.022692267100016275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,2560,0.012275200088818867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,2560,0.02035520076751709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,2560,0.010750933488210043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,2048,0.010458667079607646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,2048,0.01954560081164042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,10240,0.0353653351465861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,1536,0.00876586635907491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,1536,0.01684053341547648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,7168,0.02031466762224833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,1024,0.00693333347638448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,3072,0.011576533317565918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,6144,0.018141865730285645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,768,0.005209599932034811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,768,0.01570026675860087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,5120,0.01618346671263377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,512,0.003902933249870936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,512,0.015065600474675497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,1024,0.015762133399645488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,4096,0.01405333379904429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,256,0.00347626656293869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,3584,0.012503467003504434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,256,0.014909866452217101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,128,0.0032255999743938447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,128,0.014723199605941772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,2048,0.009501866499582927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,64,0.0030986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,64,0.01498133341471354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,5120,32,0.003110400090614955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,5120,32,0.015098667144775391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,65536,0.1916063944498698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,1536,0.00768746683994929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,65536,0.1215114672978719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,256,0.0051018665234247845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,65536,0.11731946468353271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,51200,0.1515775998433431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,51200,0.09851733048756918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,16384,0.05471146504084269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,1024,0.006520533561706543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,768,0.006183466811974844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,12288,0.04120746850967407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,16384,0.04282453457514445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,12288,0.035478401184082034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,10240,0.0352021336555481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,10240,0.031753599643707275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,8192,0.029331199328104657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,10240,0.023744000991185506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,8192,0.02797226707140605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,7168,0.026524800062179565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,7168,0.026233599583307905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,512,0.005593599875768026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,6144,0.02250986695289612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,6144,0.024676267306009927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,5120,0.018117332458496095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,5120,0.023131734132766722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,5120,128,0.004792533318201701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,4096,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,4096,0.02135466734568278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,51200,0.09340373675028482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,16384,0.03405653238296509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,3584,0.015267200271288552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,12288,0.02709439992904663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,3584,0.020798933506011964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,3072,0.011981866757074992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,3072,0.019960532585779824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,2560,0.01049066682656606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,2560,0.019731199741363524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,8192,0.019802665710449217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,2048,0.009130666653315227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,2048,0.018413867553075156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,7168,0.017570134003957114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,1536,0.007713066538174947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,1536,0.016123732924461363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,6144,0.01630506714185079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,1024,0.006293333570162455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,1024,0.015784533818562825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,5120,0.014830933014551798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,768,0.0041792000333468115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,768,0.01564479966958364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,4096,0.013018666704495748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,512,0.0036778666079044344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,512,0.015225600202878317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,3072,0.010748799641927083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,3584,0.011424000064531963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,256,0.0033642667035261786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,256,0.015026133259137472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,128,0.0031040000418821974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,128,0.014817066987355552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,2560,0.01009386678536733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,64,0.003031466652949651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,2048,0.009114666779836019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,64,0.014867200454076131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,4096,32,0.003035733352104823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,4096,32,0.014847999811172486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,1536,0.007166933516661327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,65536,0.17722986539204916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,65536,0.10861013730367024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,51200,0.13231039841969808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,51200,0.08842133680979411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,16384,0.047948801517486574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,51200,0.08609386285146078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,16384,0.04006933371225993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,12288,0.03688640197118123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,12288,0.033956265449523924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,1024,0.0060810665289560955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,10240,0.03166933258374532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,10240,0.03014933268229167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,768,0.006120533247788747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,8192,0.026469333966573076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,8192,0.026526933908462523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,512,0.005465599894523621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,7168,0.024014933904012045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,7168,0.025073067347208662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,7168,0.01648640036582947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,6144,0.019537067413330077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,6144,0.023758933941523234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,256,0.005019733309745788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,5120,0.016574933131535848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,4096,128,0.0047765334447224935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,5120,0.022358399629592896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,65536,0.10985706647237141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,4096,0.013595733046531677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,4096,0.02090880076090495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,16384,0.03187519907951355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,3584,0.012380799651145935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,3584,0.020121600230534872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,12288,0.02532586654027303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,3072,0.011060266693433126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,3072,0.019428267081578573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,10240,0.021834667523701987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,2560,0.00973653296629588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,2560,0.019001599152882895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,8192,0.01848533352216085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,2048,0.008939733107884724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,2048,0.017036799589792886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,6144,0.015675733486811318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,1536,0.007500799993673961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,1536,0.015661866466204325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,1536,0.007043200234572093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,1024,0.00496319979429245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,1024,0.015442132949829102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,768,0.004188799858093261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,768,0.015307733416557312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,768,0.006001066664854685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,512,0.003772799919048945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,5120,0.013915733496348063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,512,0.014914133151372275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,256,0.0032127998769283296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,256,0.015140266219774882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,4096,0.01230506698290507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,128,0.0030517332255840302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,128,0.01477120021979014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,3584,0.011146666606267293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,64,0.002974933385848999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,64,0.014637866616249084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,3072,0.010241066416104633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3584,32,0.0029909332593282064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3584,32,0.014724266529083253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,65536,0.15038186709086102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,2560,0.00962559978167216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,65536,0.096998397509257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,2048,0.007975466549396515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,51200,0.11974399884541828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,51200,0.08055573304494222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,16384,0.04310719966888428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,16384,0.03729386727015178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,12288,0.03365120093027751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,12288,0.032001066207885745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,1024,0.006285866598288219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,10240,0.02834239999453227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,10240,0.028725334008534747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,512,0.00544213354587555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,8192,0.024327466885248818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,256,0.004889599978923798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,8192,0.025214932362238568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,7168,0.021821866432825722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,7168,0.023870933055877685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3584,128,0.004860800007979075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,6144,0.01923840045928955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,6144,0.02286720077196757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,65536,0.11253013610839843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,5120,0.017694934209187826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,16384,0.03246399958928426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,5120,0.021865600347518922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,4096,0.012401066223780314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,4096,0.020472532510757445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,12288,0.025689599911371867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,3584,0.011213866869608562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,3584,0.019754666090011596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,10240,0.022820266087849934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,3072,0.010207999746004741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,3072,0.018897066513697304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,51200,0.09223146438598633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,2560,0.00902186632156372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,2560,0.01803626616795858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,8192,0.019127466281255088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,2048,0.007976533472537994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,2048,0.016429866353670754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,7168,0.01713493267695109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,1536,0.006849066913127899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,1536,0.015729066729545594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,6144,0.01550933321317037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,1024,0.004660266637802124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,1024,0.015845333536465965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,1024,0.00631039987007777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,768,0.004163199911514918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,768,0.015480533242225647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,5120,0.014137599865595499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,512,0.003736533224582672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,512,0.015127467115720114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,4096,0.012630400061607362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,256,0.0032448001205921174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,256,0.014805333813031516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,3584,0.010905599594116211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,128,0.0029781334102153777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,128,0.014589866995811463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,3072,0.01015786627928416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,64,0.0028970666229724885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,2560,0.009248000383377076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,64,0.014737066626548768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,3072,32,0.0028970666229724885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,3072,32,0.014833066860834757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,2048,0.007684266567230225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,65536,0.12081066767374675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,65536,0.08732799688975015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,1536,0.0066890666882197065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,51200,0.09584426879882812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,65536,0.10313493410746258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,16384,0.035131732622782394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,51200,0.06993280251820883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,16384,0.03341013193130493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,16384,0.030057599147160847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,12288,0.02797973354657491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,12288,0.028716800610224406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,10240,0.024285866816838583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,10240,0.02645866672197978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,768,0.005907199780146281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,8192,0.02029119928677877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,8192,0.02381333311398824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,512,0.0054613331953684485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,7168,0.01834026575088501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,7168,0.022884267568588256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,256,0.004891733328501383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,6144,0.0161461333433787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,6144,0.02173759937286377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,3072,128,0.004664533336957296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,5120,0.014232533176740012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,5120,0.02090453306833903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,4096,0.010981333255767823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,4096,0.01923840045928955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,51200,0.08482027053833008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,3584,0.00999679962793986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,3584,0.019191465775171914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,12288,0.02363626758257548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,3072,0.009129599730173747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,3072,0.01844053268432617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,10240,0.020834134022394816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,2560,0.008221866687138875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,2560,0.0164000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,8192,0.017401599884033205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,2048,0.0072202667593956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,2048,0.016170666615168253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,7168,0.015345066785812378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,1536,0.005894400179386139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,1536,0.016040533781051636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,1536,0.006405333181222279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,1024,0.004530133306980133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,6144,0.014443733294804893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,1024,0.015428266922632852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,768,0.004098133246103922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,768,0.01544533371925354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,5120,0.013090133666992188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,512,0.003566933423280716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,512,0.014893866578737893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,4096,0.012056533495585125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,256,0.003173333406448364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,256,0.014520532886187234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,3584,0.010288000106811523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,128,0.002906666696071625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,128,0.014673067132631936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,3072,0.009443199634552002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,64,0.002850133428970973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,64,0.01477226714293162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2560,32,0.0028927999238173166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,2560,0.00795839975277583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2560,32,0.014794666568438211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,65536,0.10147840181986492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,2048,0.0072736000021298725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,51200,0.07887893517812093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,51200,0.06149653196334839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,1024,0.006066133578618368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,16384,0.029158399502436323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,16384,0.030190932750701904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,768,0.005745066702365876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,12288,0.02345493237177531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,12288,0.02595413327217102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,512,0.005338666836420695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,10240,0.020259199539820354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,10240,0.024077866474787393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,65536,0.07365439732869467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,8192,0.01688213348388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,8192,0.022601600488026938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,256,0.00489386667807897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,8192,0.017080533504486083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,7168,0.02164693276087443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2560,128,0.004695466657479604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,6144,0.013754666845003764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,6144,0.020757333437601725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,7168,0.015146666765213012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,5120,0.012113066514333089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,65536,0.09379626909891764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,5120,0.020066134134928384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,4096,0.010517332951227825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,51200,0.0772874673207601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,4096,0.01783999999364217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,3584,0.009886933366457622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,3584,0.018196266889572144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,3584,0.009381332993507385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,16384,0.027805866797765096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,3072,0.009160533547401428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,3072,0.017347200711568197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,12288,0.022184532880783082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,2560,0.007623466849327088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,2560,0.016685867309570314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,10240,0.01985599994659424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,2048,0.006826666494210561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,2048,0.016509866714477538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,7168,0.01493333379427592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,1536,0.005161599814891815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,1536,0.015999999642372132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,6144,0.013886933525403341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,1024,0.004368000229199727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,1024,0.015552000204722086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,5120,0.012593066692352295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,768,0.003977599988381068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,768,0.015247999628384908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,4096,0.011318399508794149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,512,0.0035968000690142312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,512,0.014825600385665893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,3072,0.008566400408744812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,256,0.003218133250872294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,256,0.014722133676211039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,128,0.002865066627661387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,2560,0.008171733220418293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,128,0.014552533626556396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,64,0.0028042666614055633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,64,0.014573867122332254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,2048,32,0.0028351999819278715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,2048,32,0.01455573340257009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,2048,0.007106133302052816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,65536,0.07893439928690592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,65536,0.06316800117492676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,51200,0.0630783994992574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,51200,0.05186453262964884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,1536,0.006649599969387054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,16384,0.023681066433588662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,16384,0.026614399751027425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,1024,0.006169599791367849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,12288,0.019063466787338258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,12288,0.023756800095240276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,768,0.005865600208441416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,10240,0.016285866498947144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,10240,0.022334933280944824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,512,0.0054293334484100345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,8192,0.013581867019335428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,8192,0.021078399817148843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,256,0.0048991998036702475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,7168,0.012627200285593668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,2048,128,0.0047423998514811196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,7168,0.020579200983047486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,6144,0.011417599519093831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,6144,0.020551466941833497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,65536,0.08318719863891602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,5120,0.01037013332049052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,5120,0.019661867618560792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,5120,0.011374933520952861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,4096,0.009417600433031718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,51200,0.06683626969655355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,4096,0.017465599377950034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,3584,0.008890666564305623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,3584,0.01695466637611389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,16384,0.025130667289098102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,3072,0.00801386684179306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,3072,0.01720213294029236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,12288,0.019522132476170857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,2560,0.007181866466999054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,2560,0.016742400328318276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,10240,0.01754986643791199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,2048,0.005532800157864889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,2048,0.016562133034070333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,8192,0.01462399959564209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,1536,0.0048767998814582825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,1536,0.01600106656551361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,7168,0.013285332918167114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,1024,0.004223999877770742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,1024,0.015343999862670899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,6144,0.01255466639995575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,768,0.0038581334054470064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,768,0.015242666999499003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,4096,0.008970666925112407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,512,0.0035584000249703727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,3584,0.008564266562461852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,512,0.015200000007947287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,256,0.003147733211517334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,256,0.014729600151379904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,256,0.004804266492525736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,128,0.002869333326816559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,128,0.01461120049158732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,3072,0.007768533130486806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,64,0.002820266783237457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,64,0.014662399888038635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1536,32,0.002881066749493281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1536,32,0.01476693352063497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,65536,0.05354453325271606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,65536,0.0498858650525411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,2560,0.007674666742483775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,51200,0.04301653305689494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,51200,0.04192320108413696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,2048,0.006788266698519389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,1536,0.006246399879455566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,16384,0.017240534226099648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,1024,0.005817600091298421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,16384,0.02341759999593099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,12288,0.013799466689427695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,12288,0.021384533246358237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,768,0.00544106662273407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,10240,0.012302933136622111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,10240,0.021211733420689903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,512,0.005117866893609365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,8192,0.010808533430099488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,8192,0.020131200551986694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1536,128,0.004610133171081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,7168,0.010211199522018433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,7168,0.019604265689849854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,7168,0.012282666563987733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,6144,0.009064533313115438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,6144,0.01842666665712992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,65536,0.07841920057932536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,5120,0.009150933225949604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,5120,0.018133334318796792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,4096,0.008335999647776286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,4096,0.017633066574732462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,51200,0.06229120095570882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,3584,0.007433600227038066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,3584,0.017757866779963175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,3584,0.008150400221347808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,3072,0.006790400048096975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,16384,0.022747733195622764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,3072,0.016670932372411094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,12288,0.018473599354426065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,2560,0.0062837332487106325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,2560,0.016633599996566772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,2048,0.005543466905752818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,2048,0.01623466710249583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,10240,0.016288000345230102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,1536,0.004891733328501383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,1536,0.016499200463294984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,8192,0.013832533359527588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,1024,0.005299200117588043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,1024,0.015796266992886863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,6144,0.01107199986775716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,768,0.004541866481304169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,768,0.01581333378950755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,5120,0.009911466638247173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,512,0.0037834666669368743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,512,0.015179733435312906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,4096,0.008620799581209818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,256,0.0032757334411144257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,256,0.014777599771817525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,3072,0.007531733314196269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,128,0.0030773334205150605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,128,0.014673067132631936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,2560,0.007203199962774913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,64,0.002979200085004171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,64,0.014634666840235391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,2048,0.006436266501744588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,1024,32,0.003089066594839096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,1024,32,0.015033599734306336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,65536,0.04245973428090413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,65536,0.04405973354975383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,51200,0.03454826672871907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,1536,0.006043733159701029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,51200,0.037323733170827225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,16384,0.014062933127085366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,16384,0.022182399034500123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,1024,0.005382399757703145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,12288,0.011746133367220562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,12288,0.020541866620381675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,768,0.005128533144791921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,10240,0.010894933342933654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,10240,0.020266666014989217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,512,0.004856533308823904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,8192,0.00960533320903778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,8192,0.01845759948094686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,256,0.004704000055789947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,7168,0.009084799885749817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,7168,0.018984532356262206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,1024,128,0.004609066744645437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,6144,0.008398933211962382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,6144,0.01969919999440511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,65536,0.0777237335840861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,5120,0.007586133480072021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,51200,0.06165973345438639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,5120,0.01934079925219218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,4096,0.006562133133411407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,4096,0.017930666605631508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,16384,0.022797866662343343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,3584,0.007782400151093801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,3584,0.017093332608540852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,12288,0.018281600872675576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,3072,0.007216000060240428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,3072,0.016833066940307617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,10240,0.016125866770744325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,2560,0.006227200229962667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,2560,0.016475733121236166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,8192,0.01304533382256826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,2048,0.0054613331953684485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,2048,0.016556800405184428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,7168,0.011821867028872172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,6144,0.010851200421651204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,1536,0.004790399968624115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,1536,0.015943466623624166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,5120,0.009726933638254802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,1024,0.004123733441034953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,1024,0.01544533371925354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,4096,0.008641067147254943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,768,0.003752533346414566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,3584,0.00806826651096344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,768,0.015215999881426492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,512,0.00344106654326121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,512,0.015434666474660238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,3072,0.007524266839027405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,256,0.0031680000325044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,256,0.014526933431625366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,2560,0.007191466788450877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,128,0.0029397333661715193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,128,0.014523733655611673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,2048,0.006411733229955037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,64,0.002739199995994568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,64,0.014339199662208557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,768,32,0.0027327999472618104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,768,32,0.014485333363215128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,65536,0.030793599287668866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,65536,0.03712746699651082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,51200,0.025360000133514405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,51200,0.03089173237482707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,51200,0.061027201016743984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,16384,0.010812800129254658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,16384,0.020240000883738198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,1536,0.0058794667323430385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,12288,0.009562666217486065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,12288,0.018849066893259683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,1024,0.0052704001466433205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,10240,0.008726400136947633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,10240,0.01874133348464966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,768,0.00513919989267985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,8192,0.007885866860548655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,8192,0.018518400192260743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,512,0.004808533191680908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,7168,0.007214933137098948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,7168,0.020454400777816774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,256,0.004748799900213877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,6144,0.006623999774456024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,6144,0.019349332650502524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,768,128,0.004509866734345754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,5120,0.0066431999206542965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,5120,0.01886826753616333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,65536,0.07746240297953287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,4096,0.006267733375231425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,4096,0.017650133371353148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,16384,0.022507733106613158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,3584,0.007367466886838277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,3584,0.017382399241129557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,12288,0.01772693395614624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,3072,0.006692266464233399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,3072,0.016852267583211265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,10240,0.015288533767064414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,2560,0.006045866509278615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,2560,0.016472533345222473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,8192,0.012845866878827415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,2048,0.005340800185998281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,2048,0.016204800208409628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,7168,0.011755733688672384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,1536,0.004779733220736186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,1536,0.015756799777348836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,6144,0.010637866457303365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,1024,0.0041184000670909885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,1024,0.015397333105405173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,5120,0.00969599982102712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,768,0.003740799923737844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,768,0.015148799618085226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,4096,0.00860053300857544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,512,0.003337600082159042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,512,0.014912000298500061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,3584,0.008036266764005024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,256,0.003070933371782303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,256,0.014506666858990987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,3072,0.007422933479150136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,128,0.0028223998844623564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,128,0.01446293294429779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,2560,0.007032533486684163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,64,0.00277866671482722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,2048,0.006270933151245117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,64,0.01437333325544993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,512,32,0.002703999976317088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,65536,0.018427733580271402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,65536,0.029415466388066608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,1536,0.0060032000144322716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,51200,0.01470186710357666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,51200,0.025380265712738038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,51200,0.06126399834950765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,1024,0.005272533496220907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,16384,0.008065066734949748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,16384,0.019355734189351402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,12288,0.008006399869918824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,12288,0.018947199980417887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,768,0.005138133466243744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,10240,0.0071712002158164975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,512,32,0.01458133359750112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,10240,0.018703999121983846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,512,0.0047199999292691554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,8192,0.0065194666385650635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,8192,0.018014933665593466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,256,0.004598399996757508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,7168,0.006247466802597046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,7168,0.01844480037689209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,512,128,0.004517333209514618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,6144,0.006089599927266439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,6144,0.018074667453765868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,65536,0.07683733304341635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,5120,0.006494933366775512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,5120,0.01858773430188497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,16384,0.02174826661745707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,4096,0.006109866499900818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,12288,0.017333332697550455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,4096,0.01797119975090027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,3584,0.0065184002121289565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,3584,0.017358932892481485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,10240,0.015211733182271323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,3072,0.0062047998110453285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,3072,0.016740266482035318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,2560,0.006077866752942403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,2560,0.016269866625467935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,8192,0.012825600306193032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,2048,0.005416533350944519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,2048,0.01633386711279551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,7168,0.011640533804893494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,1536,0.004722133278846741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,1536,0.01604586640993754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,6144,0.0107232004404068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,1024,0.004070399949947992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,1024,0.015400532881418863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,5120,0.009701333443323771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,768,0.0036938667297363283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,768,0.015135999520619711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,4096,0.008640000224113464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,512,0.003433600068092346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,512,0.014776532848676046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,3584,0.008088533580303193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,256,0.002976000060637792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,256,0.014734933773676554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,3072,0.007482666770617168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,128,0.0029333333174387617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,128,0.014486400286356607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,2560,0.0070816000302632645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,64,0.00266239990790685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,64,0.014425599575042724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,2048,0.006382933259010315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,256,32,0.01439466675122579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,1536,0.005834666887919108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,65536,0.011737599968910217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,65536,0.025534933805465697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,51200,0.010360532999038696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,51200,0.022717867294947305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,1024,0.005191466708978017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,16384,0.006687999765078227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,16384,0.018887466192245482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,256,32,0.0026517334083716077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,12288,0.006175999840100607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,12288,0.018618667125701906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,12288,0.0173962672551473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,10240,0.006905599931875865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,768,0.0049898669123649595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,10240,0.018669867515563966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,8192,0.006376533210277558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,8192,0.01773973306020101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,512,0.004762666424115499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,7168,0.0061930666367212926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,7168,0.018557866414388023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,256,0.004599466423193614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,6144,0.005961599946022034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,6144,0.018309332927068076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,256,128,0.004481066763401031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,5120,0.006346666812896728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,5120,0.01839146614074707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,65536,0.07616746425628662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,4096,0.005949866771697998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,4096,0.01762239933013916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,51200,0.0602570652961731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,3584,0.006367999811967213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,3584,0.017130666971206666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,16384,0.021713066101074218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,3072,0.006093866626421611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,3072,0.016999467213948568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,10240,0.01511573294798533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,2560,0.006067200005054474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,2560,0.01698026657104492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,8192,0.01285546620686849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,2048,0.00535999983549118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,2048,0.016684800386428833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,7168,0.011655466755231221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,1536,0.004691199958324432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,1536,0.016082132856051125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,6144,0.010729599992434185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,1024,0.004078933348258337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,1024,0.015334399541219077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,5120,0.009635200103123982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,768,0.003732266773780187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,4096,0.008543999989827473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,768,0.01495573321978251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,768,0.005047466854254405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,3584,0.008141866823037466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,512,0.003310933212439219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,512,0.014840533336003622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,256,0.0029674666623274487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,256,0.01458453337351481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,3072,0.007365333537260692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,128,0.002887466549873352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,128,0.014553599556287131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,2560,0.007107200225194295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,64,0.002648533384005229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,64,0.014477866888046264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,128,32,0.0026410666604836782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,128,32,0.014539733529090881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,65536,0.010477866729100544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,1536,0.005814399818579356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,65536,0.02405760089556376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,51200,0.00902400016784668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,51200,0.0222762664159139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,16384,0.006589866677920024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,12288,0.006027733286221823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,16384,0.01885546644528707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,10240,0.006312533219655354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,12288,0.018462934096654258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,8192,0.006221866607666016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,10240,0.01879040002822876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,1024,0.005303466816743215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,7168,0.0060586666067441305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,8192,0.01808639963467916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,6144,0.005854933460553487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,5120,0.0062165334820747375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,6144,0.018049067258834837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,4096,0.005846400062243144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,5120,0.01844373345375061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,3584,0.006355200211207073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,4096,0.01744106610616048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,3072,0.0060928001999855045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,3584,0.017233065764109292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,2560,0.00625493327776591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,3072,0.016780799627304076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,2048,0.005393066505591074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,2560,0.016317866245905557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,1536,0.004634666442871094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,2048,0.015838932991027833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,1024,0.004078933348258337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,1536,0.015982932845751443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,768,0.003706666578849157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,1024,0.015634133418401083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,512,0.0033215999603271483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,768,0.015085867047309876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,256,0.0029845332105954488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,512,0.01486186683177948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,256,0.014577066898345948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,128,0.003028266628583272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,64,0.0027232001225153605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,128,0.014587733149528503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,64,32,0.002674133330583572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,64,0.014550399780273438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,32,0.014472533265749613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,512,0.0048426667849222815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,65536,0.0240938663482666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,51200,0.01064639985561371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,51200,0.02261013388633728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,256,0.004574933151404063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,16384,0.0067210664351781205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,16384,0.019006933768590292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,12288,0.006025599936644236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,10240,0.006260266900062561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,8192,0.006232533355553945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,7168,0.00598826656738917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,10240,0.019156267245610557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,8192,0.01792959968249003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,6144,0.005835733314355215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,7168,0.01833066741625468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,6144,0.01781760056813558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,5120,0.006200533111890157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,4096,0.005809066692988077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,5120,0.01835626761118571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,64,7168,0.018424532810846963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,3584,0.006323199967543285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,3072,0.005965866645177205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,3584,0.017128533124923705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,2560,0.006004266440868378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,3072,0.016993065675099693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,2560,0.016311466693878174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,2048,0.0053151999910672505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,1536,0.0047199999292691554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,2048,0.01620693306128184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,1024,0.003976533313592275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,1536,0.015731199582417806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,768,0.00360000009338061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,512,0.003307733436425527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,1024,0.015299200018246969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,768,0.015262933572133383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,256,0.003019733230272929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,512,0.014794666568438211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,128,0.002809600035349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,256,0.014677332838376364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,128,0.014524799585342408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,64,0.0026186667382717133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,32,0.002625066787004471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,64,0.014509866635004679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,2048,0.006268799801667531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,32,0.01437333325544993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,49,32,65536,0.012840533256530761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,12288,0.01851200064023336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,49,128,128,0.004568533102671305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,16384,0.3988032023111979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,16384,0.7525653203328451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,12288,0.3038784027099609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,12288,0.5700874964396159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,49,32,4096,0.01758400003115336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,51200,1.2028138478597006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,10240,0.25504533449808753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,10240,0.47811412811279297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,10240,0.2691754659016927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,51200,2.3482081095377607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,8192,0.20848320325215658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,8192,0.3813920021057129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,7168,0.16552106539408368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,7168,0.18438506126403809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,6144,0.28924907048543297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,7168,0.39292373657226565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,6144,0.16119680404663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,6144,0.14402346611022948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,5120,0.24137813250223794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,5120,0.15726186434427897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,4096,0.2003551959991455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,4096,0.11766719818115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,3584,0.10122133096059163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,3584,0.1711466630299886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,16384,0.3666794776916504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,3584,0.08803199927012126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,3072,0.14795519510904948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,3072,0.08998186588287353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,2560,0.12460906505584717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,2560,0.07849706808725992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,2048,0.10034666856129963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,51200,1.131780242919922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,2048,0.06620373328526816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,1536,0.07750826676686605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,2048,0.060893865426381436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,12288,0.2771327972412109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,1536,0.05490133364995321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,1024,0.06154239972432455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,1024,0.04248960018157959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,768,0.04247573216756185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,768,0.03653333187103271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,8192,0.1881663958231608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,512,0.033847467104593916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,512,0.029227733612060547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,5120,0.12176746527353924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,256,0.019317332903544107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,4096,0.09984320004781087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,256,0.021760000785191854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,128,0.0100000003973643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,128,0.019384533166885376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,3072,0.07730666796366373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,64,0.008110933502515157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,64,0.017607466379801432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,32,0.005876266459623972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,2560,0.06558613379796346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,32,0.01773653427759806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,1536,0.04239786863327026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,1024,0.03158826629320781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,768,0.025603200991948443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,512,0.020102399587631225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,65536,1.205144500732422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,256,0.014566399653752646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,128,0.012056533495585125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,51200,0.9400981267293295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,16384,0.3169941266377767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,16384,0.6032874425252278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,65536,2.379515838623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,51200,1.8642932891845703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,12288,0.47416213353474934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,12288,0.24315519332885743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,10240,0.18335466384887694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,10240,0.43469546635945633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,10240,0.23245973587036134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,8192,0.3019359906514486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,8192,0.16749547322591146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,7168,0.14921493530273439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,7168,0.26442346572875974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,6144,0.13066879908243817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,6144,0.22600852648417152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,5120,0.19399785995483398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,5120,0.11168746948242188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,65536,1.1279691060384116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,16384,0.29053866068522133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,4096,0.17711893717447919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,51200,0.8809194564819336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,4096,0.09337066809336345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,3584,0.08319786389668783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,3584,0.15766506195068358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,12288,0.2546208063761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,3072,0.1336949348449707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,3072,0.06106880108515421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,3072,0.07707306543986002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,2560,0.09903573195139567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,2560,0.06475840012232462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,2560,0.05335359970728556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,8192,0.17154134114583333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,2048,0.07901120185852051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,7168,0.13394559224446614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,2048,0.05524693330128988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,1536,0.061465601126352944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,1536,0.04603093465169271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,1024,0.042200533548990886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,1024,0.03626453479131063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,6144,0.11393067042032878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,1024,0.026215465863545735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,768,0.037034666538238524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,768,0.03089066743850708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,512,0.024989867210388185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,512,0.02509653369585673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,5120,0.09655893643697103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,256,0.0145087997118632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,256,0.02055786649386088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,4096,0.07941653728485107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,128,0.008665600419044494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,128,0.016772266228993735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,3584,0.0702186663945516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,128,0.009165866176287334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,64,0.0057087997595469155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,64,0.017154133319854735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,51200,32,0.005374933282534281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,51200,32,0.017126399278640746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,65536,0.3842207908630371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,65536,0.7494400024414063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,2048,0.04859306812286377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,1536,0.03455146551132202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,16384,0.18639146486918132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,768,0.021364265680313112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,51200,0.589799435933431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,51200,0.32484480539957683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,16384,0.11043946743011475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,51200,0.28790400822957357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,512,0.01633280018965403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,12288,0.09895040194193522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,12288,0.1648608048756917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,10240,0.12969919840494792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,10240,0.07530559698740641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,8192,0.09563519954681396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,8192,0.06352959871292115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,7168,0.08374186356862387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,51200,256,0.012286933263142903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,7168,0.05807786782582601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,6144,0.07335039774576822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,6144,0.05203946828842163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,5120,0.061465601126352944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,5120,0.04597333272298177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,65536,0.3688117345174154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,4096,0.058354131380716955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,4096,0.03981333176294963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,12288,0.07537919680277506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,16384,0.09836800098419189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,3584,0.04437226851781209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,3584,0.037215999762217206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,10240,0.0639135996500651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,3072,0.03882453441619873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,3072,0.03371946811676026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,8192,0.052604798475901285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,2560,0.03300693432490031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,2560,0.03101866642634074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,2048,0.027159466346104937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,7168,0.04708693424860637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,2048,0.027451733748118084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,2048,0.01795413295427958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,6144,0.04152213335037232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,1536,0.024119466543197632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,1024,0.015187199910481772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,1024,0.020808533827463786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,5120,0.03573973178863525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,768,0.011875200271606445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,768,0.01941439906756083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,1536,0.02156160076459249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,512,0.009005866448084513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,512,0.018151466051737467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,4096,0.030044800043106078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,256,0.006054399907588959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,256,0.01622719963391622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,3584,0.027194666862487792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,128,0.0038805333276589714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,128,0.015406933426856995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,3072,0.023996800184249878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,64,0.003640533238649368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,2560,0.02132479945818583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,64,0.015412267049153647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,32,0.003689600030581156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,32,0.01573973298072815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,1536,0.014857600132624308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,1024,0.011608533064524333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,768,0.010232533017794292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,65536,0.3079648017883301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,65536,0.574732780456543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,51200,0.4405952135721843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,16384,0.15271786053975422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,51200,0.2458869298299154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,16384,0.10355626742045085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,12288,0.11573226451873779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,12288,0.07222506999969483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,512,0.008707200487454731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,10240,0.0976095994313558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,10240,0.06303573449452718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,256,0.006163200239340464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,8192,0.07710506916046142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,8192,0.053717335065205894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,128,0.0059232001503308615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,7168,0.06516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,7168,0.04783253272374471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,65536,0.2799071947733561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,6144,0.06296639839808146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,51200,0.22000853220621744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,6144,0.04495893319447835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,5120,0.039683198928833006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,5120,0.04859093427658081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,4096,0.0406335989634196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,16384,0.07589013576507568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,4096,0.03487253189086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,3584,0.035785599549611406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,4096,0.02617173393567403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,3584,0.03235093355178833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,3072,0.031188267469406127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,3072,0.030102399984995525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,12288,0.0581055998802185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,2560,0.027638399600982667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,2560,0.027155200640360515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,2560,0.016921599706014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,2048,0.022175999482472737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,2048,0.024757333596547446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,10240,0.04955306847890218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,1536,0.017391999562581383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,1536,0.022755199670791627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,8192,0.04143253167470296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,1024,0.012602667013804117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,1024,0.019640533129374187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,1024,0.00978773335615794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,768,0.010058666268984478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,768,0.018413867553075156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,7168,0.03674346605936686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,512,0.0075914666056633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,512,0.015733333428700765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,6144,0.03669439951578776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,5120,0.02818453311920166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,256,0.004102399945259095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,256,0.015608533223470052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,128,0.0036831999818483984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,128,0.015314132968584696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,3584,0.021556266148885093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,64,0.003366400053103765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,64,0.015056000153223673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,32,0.003446399917205175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,32,0.01534293293952942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,2048,0.014167466759681701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,1536,0.011924266815185547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,65536,0.48227732976277665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,65536,0.26744747161865234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,51200,0.3787925402323405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,51200,0.2141589323679606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,3072,0.01914986570676168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,768,0.008619733651479085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,16384,0.0748693307240804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,512,0.006226133306821187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,12288,0.08834773699442545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,12288,0.06110613346099854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,10240,0.0739135980606079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,10240,0.053483732541402186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,256,0.0056085333228111265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,8192,0.06041173140207926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,128,0.005291733145713806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,8192,0.046404266357421876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,7168,0.05382080078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,7168,0.04249173402786255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,65536,0.23751360575358071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,6144,0.04696746667226155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,16384,0.1303381363550822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,6144,0.03909866809844971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,5120,0.03987520138422648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,5120,0.03485333522160848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,16384,0.06508373419443766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,12288,0.05012266635894776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,4096,0.03368959824244182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,51200,0.18708693186442057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,4096,0.030857600768407184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,3584,0.028967465957005816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,3584,0.03283093372980754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,3072,0.02547840078671773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,3072,0.026553599039713542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,8192,0.03563520113627116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,2560,0.021949867407480873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,2560,0.02481386661529541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,7168,0.03181866606076558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,2048,0.018170666694641114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,10240,0.042606933911641436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,2048,0.022795732816060385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,1536,0.01416106621424357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,1536,0.02122453252474467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,6144,0.02916053334871928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,2560,0.014845866958300272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,5120,0.024420267343521117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,1024,0.010532266894976298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,1024,0.01880426605542501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,768,0.008730666836102803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,768,0.017193599541982015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,4096,0.020729599396387736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,512,0.007008000214894612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,512,0.015585066874821982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,3072,0.01646719972292582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,256,0.003920000046491623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,256,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,3584,0.018632533152898152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,128,0.0034805332620938623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,2048,0.012436266740163167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,64,0.0032874666154384612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,64,0.015222400426864624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,32,0.003311999887228012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,32,0.015271466970443726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,1536,0.010694400469462077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,65536,0.3825770696004232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,1024,0.00902826686700185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,128,0.015075199802716575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,65536,0.2123701254526774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,768,0.0073066666722297665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,16384,0.0934005339940389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,512,0.00590826670328776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,51200,0.1701258659362793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,16384,0.06356586615244547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,51200,0.3109482765197754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,12288,0.07146346569061279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,12288,0.05217066605885824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,10240,0.06044053236643473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,10240,0.04588586489359538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,256,0.005298133194446564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,8192,0.04917973279953003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,8192,0.04007360140482585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,7168,0.04332480033238729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,7168,0.037273601690928145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,128,0.005100800096988678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,6144,0.03848533233006795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,6144,0.03377386728922526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,10240,0.035589333375295004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,5120,0.032333866755167646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,5120,0.030561065673828124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,5120,0.020731733242670695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,4096,0.026679466168085735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,4096,0.027382399638493853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,65536,0.1918794631958008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,3584,0.027036799987157183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,3584,0.02574293414751689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,51200,0.150983460744222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,16384,0.053291734059651694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,3072,0.023628799120585124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,3072,0.014465066790580749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,3072,0.024101332823435465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,12288,0.047329068183898926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,2560,0.018075732390085857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,2048,0.014762666821479798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,2560,0.02257386644681295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,2048,0.020980266729990642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,8192,0.029690666993459063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,1536,0.019556266069412232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,7168,0.026629332701365156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,1024,0.008840533097585042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,1024,0.0180842657883962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,6144,0.023794132471084594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,768,0.007643733421961467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,768,0.015386666854222616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,1536,0.01172160009543101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,512,0.00591786652803421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,512,0.015631999572118124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,4096,0.017667200167973837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,256,0.0037130666275819145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,256,0.015264000495274863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,3584,0.016196266810099284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,128,0.003453866640726725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,128,0.014995200435320535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,2560,0.013124266266822815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,64,0.003316266586383184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,64,0.015072000026702882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,2048,0.011374933520952861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,32,0.0033813332517941795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,1536,0.009831466277440389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,32,0.015085867047309876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,1024,0.008157866696516674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,65536,0.2015530745188395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,65536,0.3383946736653646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,51200,0.16389865875244142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,16384,0.08837546507517496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,16384,0.057885865370432533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,16384,0.04876266717910767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,768,0.006698666512966156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,12288,0.06319359938303629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,12288,0.04845013221104939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,10240,0.05332693258921305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,10240,0.042637864748636886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,512,0.005827199916044871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,8192,0.043443198998769125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,256,0.005376000205675761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,8192,0.037069865067799884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,51200,0.2683712005615234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,7168,0.04420160055160523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,7168,0.03453973531723022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,7168,0.024499199787775674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,6144,0.03416106700897217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,6144,0.03257386684417725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,128,0.005351466437180838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,5120,0.029015467564264937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,5120,0.029204267263412475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,65536,0.17517120043436687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,4096,0.027048534154891966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,51200,0.14272319475809733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,4096,0.025407999753952026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,3584,0.021559466918309532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,3584,0.024229333798090617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,3584,0.014686933159828186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,3072,0.018952532609303793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,3072,0.023063466946283976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,12288,0.038285867373148603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,2560,0.01629866659641266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,2560,0.021873066822687783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,10240,0.03272213339805603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,2048,0.013505066434542337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,2048,0.020296533902486168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,8192,0.027368533611297607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,1536,0.011037866274515789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,1536,0.019411200284957887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,1536,0.00918506681919098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,1024,0.008252800007661184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,1024,0.016782933473587038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,768,0.007051733136177063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,768,0.01564586659272512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,6144,0.021848533550898233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,512,0.004558933277924856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,512,0.015433599551518759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,512,0.005467733244101206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,256,0.0036608000596364343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,256,0.01508799990018209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,5120,0.018863999843597413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,128,0.003286399940649668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,128,0.01480959951877594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,4096,0.01797440052032471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,64,0.003123199939727783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,64,0.014993066589037577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,32,0.0031786667803923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,32,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,3072,0.012985600034395852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,2560,0.01193386713663737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,65536,0.28809601465861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,65536,0.17183146476745606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,2048,0.010466133554776508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,51200,0.22751572926839195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,51200,0.13824853897094727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,1024,0.007114666700363159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,16384,0.09176106452941894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,16384,0.05531413157780966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,12288,0.0626698652903239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,12288,0.04464213450749715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,768,0.00626986672480901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,10240,0.05247360070546468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,10240,0.03983253240585327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,8192,0.04053440093994141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,8192,0.03562879959742228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,256,0.005266133447488149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,7168,0.03548373381296794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,128,0.004940799872080485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,7168,0.033353598912556966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,6144,0.031062400341033934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,6144,0.03054186701774597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,65536,0.1553237279256185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,5120,0.026292266448338826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,5120,0.02763413389523824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,51200,0.12338346640268963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,4096,0.02168853282928467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,4096,0.02480319937070211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,12288,0.03428053458531698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,3584,0.019703465700149535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,3584,0.023735467592875162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,10240,0.02956266601880391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,3072,0.01739306648572286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,3072,0.022450133164723714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,8192,0.024665600061416625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,2560,0.01503679951032003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,2560,0.02144213318824768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,7168,0.02221333384513855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,2048,0.012621866663297019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,2048,0.01993066668510437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,6144,0.019682133197784425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,1536,0.010410666465759277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,16384,0.04352426528930664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,1536,0.018992000818252565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,1024,0.007818666597207386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,1024,0.016126933693885803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,5120,0.016985599199930826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,768,0.00655680000782013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,768,0.015280000368754067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,4096,0.01452906628449758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,512,0.0040277334551016486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,3584,0.013244799772898355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,512,0.01530239979426066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,3072,0.012008532881736755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,256,0.0035146666069825493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,2560,0.011059199770291645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,256,0.015017599860827128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,128,0.0032277333239714304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,128,0.014914133151372275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,2048,0.009577600161234538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,1536,0.00843519965807597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,64,0.0030879999200503034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,32,0.0031541332602500914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,64,0.014971733093261719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,32,0.014915200074513755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,1024,0.006363733112812043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,65536,0.2522570610046387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,65536,0.1502954641977946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,51200,0.20105600357055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,768,0.006060799956321717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,16384,0.06564266681671142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,16384,0.04936106602350871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,51200,0.11560746828715007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,12288,0.05113173325856527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,12288,0.03917119900385539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,10240,0.04397120078404744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,512,0.005464533468087515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,10240,0.035266133149464925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,8192,0.03573760191599528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,256,0.005038933455944061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,8192,0.0313482662041982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,7168,0.028716800610224406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,7168,0.029330132404963176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,128,0.004820266862710317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,7168,0.019836799303690592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,6144,0.025916800896326704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,6144,0.02685653368631999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,6144,0.01754986643791199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,5120,0.021871999899546305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,5120,0.024842667579650878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,4096,0.01829866568247477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,4096,0.022705066204071044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,4096,0.01328000028928121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,3584,0.016297599673271178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,3584,0.021769599119822184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,65536,0.1380949338277181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,3072,0.014275200168291726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,3072,0.02123840053876241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,51200,0.10970773696899414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,2560,0.012349866827329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,2560,0.020104533433914183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,16384,0.03876906633377075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,2048,0.010544000069300334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,12288,0.03056640028953552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,2048,0.018953599532445273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,2048,0.009115733702977498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,1536,0.00886079967021942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,1536,0.017564799388249716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,1024,0.006921599805355072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,10240,0.02638186613718669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,1024,0.015703466534614564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,768,0.0059797331690788266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,768,0.01554026703039805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,8192,0.02209386626879374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,512,0.003902933249870936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,512,0.015251200397809347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,5120,0.015348266561826071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,256,0.003390933324893316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,256,0.014973866939544677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,3584,0.01220373312632243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,128,0.0032757334411144257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,128,0.014782933394114175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,128,0.004830933113892873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,64,0.0030826665461063385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,3072,0.011165866255760193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,64,0.014793599645296732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,32,0.014866133530934652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,2560,0.010276266932487487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,65536,0.12071786721547444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,32,0.0031210665901501974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,1536,0.007709866762161255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,51200,0.1508778731028239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,1024,0.00626453310251236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,51200,0.09789226849873861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,65536,0.19333546956380207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,51200,0.09253653685251871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,16384,0.05197439988454183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,16384,0.04235733350118001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,768,0.005948799848556519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,12288,0.03556053241093953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,10240,0.03480639855066935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,512,0.005340800185998281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,10240,0.031574400266011556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,8192,0.028888533512751263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,8192,0.028043733040491743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,8192,0.018618667125701906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,7168,0.0261898676554362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,7168,0.016702934106191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,7168,0.025770666201909383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,6144,0.021492266654968263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,256,0.004930133124192556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,12288,0.040788265069325765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,5120,0.01830186645189921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,5120,0.02305813431739807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,6144,0.024642133712768556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,65536,0.11373013655344646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,4096,0.015213867028554281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,4096,0.021715199947357176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,3584,0.013515733679135642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,3584,0.020641066630681357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,12288,0.025646932919820148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,3072,0.012027733524640401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,16384,0.03609600067138672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,3072,0.02021119991938273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,2560,0.010427733262379963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,3072,0.010202667117118836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,10240,0.022035199403762817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,2560,0.019537067413330077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,2048,0.009103999535242716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,2048,0.018151466051737467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,1536,0.007704533139864604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,1536,0.015995732943216958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,6144,0.014940800269444785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,1024,0.006106666723887126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,1024,0.01565120021502177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,768,0.0040554667512575785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,768,0.015237333377202353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,5120,0.013337600231170654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,512,0.00360000009338061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,512,0.01532266636689504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,512,0.0052490666508674625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,256,0.0033461332321166994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,4096,0.011847466230392456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,256,0.01483519971370697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,128,0.0030858665704727173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,128,0.014622933665911355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,3584,0.011121066411336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,64,0.002963199963172277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,64,0.01469013293584188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,32,0.0031008000175158186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,32,0.014870400230089823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,65536,0.17661120096842448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,65536,0.11381973425547283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,65536,0.10510400136311848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,51200,0.14001065889994305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,2560,0.009607467055320739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,2048,0.008429867029190064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,51200,0.09271252950032552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,16384,0.045932801564534505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,1536,0.007004799942175548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,16384,0.039803731441497806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,12288,0.03651626507441203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,12288,0.03352320194244385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,10240,0.031221334139506025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,1024,0.006108800073464712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,10240,0.029874134063720702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,8192,0.026143999894460042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,8192,0.025941334168116253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,768,0.005975466469923655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,7168,0.023763199647267662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,7168,0.02485760052998861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,256,0.004970666766166687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,6144,0.019432532787322997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,6144,0.023475199937820435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,128,0.004875733455022176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,5120,0.016350932916005454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,5120,0.02214933236440023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,51200,0.08729813098907471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,16384,0.030101333061854047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,4096,0.013565867145856222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,4096,0.020731733242670695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,3584,0.012165333827336628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,10240,0.020482132832209267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,3584,0.020167466004689535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,3072,0.010935466488202412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,3072,0.019577600558598838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,8192,0.017268266280492148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,2560,0.00962559978167216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,2560,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,7168,0.01544319987297058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,2048,0.008382933338483174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,2048,0.017352533340454102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,12288,0.02399359941482544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,1536,0.007467733323574066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,1536,0.01590506633122762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,6144,0.014176000157992044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,1024,0.005107200145721436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,1024,0.01591679950555166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,5120,0.012698666254679362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,768,0.004075733323891958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,768,0.01532266636689504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,4096,0.011336533228556316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,512,0.003621333340803782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,3584,0.010624000430107116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,512,0.015129599968592325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,256,0.003302400062481562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,3072,0.009677867094675701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,256,0.01480959951877594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,128,0.003044266750415166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,128,0.014637866616249084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,2048,0.007357866565386455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,1536,0.006774400174617767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,64,0.0029567999144395193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,1024,0.00589333325624466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,64,0.014754133423169455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,32,0.0029824001093705496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,32,0.014616533120473226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,2560,0.009116799632708231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,65536,0.14991146723429363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,65536,0.1009002685546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,51200,0.11917013327280682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,51200,0.08154133160909018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,16384,0.042795733610788984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,16384,0.03709333340326945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,16384,0.030593067407608032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,12288,0.03263360063234965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,12288,0.03177599906921387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,768,0.005843199789524078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,10240,0.028376533587773638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,10240,0.029125332832336426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,128,0.004827733337879181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,8192,0.024396799008051553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,8192,0.0265173335870107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,7168,0.021849600474039714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,7168,0.023830399910608927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,7168,0.015686399737993875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,6144,0.01928640007972717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,6144,0.02257066567738851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,6144,0.014042666554450989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,5120,0.016643200318018594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,5120,0.021477333704630532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,512,0.005269333223501841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,4096,0.012135466933250428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,4096,0.020154666900634766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,256,0.004917333523432413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,3584,0.011188266674677531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,65536,0.10840426286061604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,3584,0.01957013408342997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,3072,0.01016533374786377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,3072,0.018913066387176512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,51200,0.08664746284484863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,2560,0.009884799520174663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,2560,0.01806186636288961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,2560,0.00869866708914439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,2048,0.007913599908351897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,2048,0.016680532693862916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,12288,0.02390506664911906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,1536,0.007005866865317028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,1536,0.016323199868202208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,1536,0.006658133367697399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,1024,0.00491946687301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,1024,0.015427199999491372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,10240,0.020541866620381675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,768,0.004053333401679992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,768,0.015449600418408713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,768,0.005763199925422668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,512,0.00359253336985906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,512,0.015098667144775391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,8192,0.017214934031168617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,256,0.0032799998919169106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,256,0.01470080018043518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,5120,0.012820266683896384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,128,0.00297173336148262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,128,0.014533332983652749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,4096,0.01120746632417043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,64,0.002916266769170761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,64,0.014613333344459533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,3584,0.010387200117111205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,32,0.0029567999144395193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,32,0.014679466684659323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,65536,0.12879786491394044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,3072,0.009276800354321798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,65536,0.0870741367340088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,65536,0.1020085334777832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,51200,0.06902613639831542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,51200,0.10232640107472737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,16384,0.03507200082143148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,2048,0.00727893312772115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,16384,0.033411200841267905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,12288,0.02773333390553792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,12288,0.028914133707682293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,10240,0.024297600984573363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,10240,0.02593066692352295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,10240,0.019197867314020792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,1024,0.005851733187834421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,8192,0.020121600230534872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,8192,0.023681066433588662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,7168,0.018169599771499633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,7168,0.02289386590321859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,7168,0.01504533290863037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,6144,0.015964800119400026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,6144,0.021901865800221763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,6144,0.013489066561063131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,5120,0.01402346690495809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,5120,0.020568533738454183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,512,0.005067733426888784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,4096,0.012128000458081562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,4096,0.019452800353368126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,256,0.004897066454092661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,3584,0.011356799801190694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,3584,0.018787199258804323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,128,0.004721066852410635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,3584,0.010146133104960124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,3072,0.009236266215642292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,3072,0.017691733439763387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,16384,0.028585600852966308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,2560,0.008364799618721008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,51200,0.08084266980489095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,12288,0.022104533513387044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,2560,0.01649173299471537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,2048,0.00726506660381953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,2048,0.017107200622558594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,1536,0.0060693333546320595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,1536,0.016145066420237223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,8192,0.01639786660671234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,1024,0.004466133316357931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,1024,0.01606826682885488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,5120,0.012236799796422322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,768,0.0040287998815377556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,768,0.015202132860819497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,768,0.005658666789531708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,512,0.0035829332967599234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,512,0.015186132987340293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,4096,0.010919466614723206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,256,0.0032586666444937387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,256,0.0146506667137146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,128,0.00297173336148262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,128,0.0144405335187912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,3072,0.008697600166002909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,64,0.002841600030660629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,64,0.014642133315404256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,32,0.0028970666229724885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,32,0.014573867122332254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,65536,0.10119679768880208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,2048,0.007055999835332234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,2560,0.007891199986139933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,65536,0.07280213038126628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,51200,0.07840533256530761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,1536,0.006237866481145223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,16384,0.028827732801437377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,51200,0.060344533125559485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,16384,0.02950719992319743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,12288,0.023450666666030885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,12288,0.025804799795150758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,1024,0.005745066702365876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,10240,0.02031893332799276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,10240,0.02403093377749125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,512,0.005126399795214335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,8192,0.016643200318018594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,8192,0.022631466388702393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,256,0.004874666531880697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,7168,0.015030399958292643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,7168,0.021729065974553426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,7168,0.013398399949073792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,6144,0.013666133085886637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,6144,0.020846933126449585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,128,0.004673066735267639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,5120,0.012095999717712403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,5120,0.01984213391939799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,65536,0.08993600209554037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,4096,0.010603732864061991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,51200,0.0739530642827352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,4096,0.019014400243759156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,16384,0.025202133258183795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,3584,0.009892266988754273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,3584,0.017679999272028603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,3072,0.008796800176302593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,12288,0.020106667280197145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,3072,0.01712426741917928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,2560,0.0077344000339508055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,2560,0.01651413341363271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,10240,0.017189333836237587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,2048,0.006865066786607106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,2048,0.016160000363985697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,8192,0.014638933539390563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,1536,0.005126399795214335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,1536,0.015897599856058757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,6144,0.012556800246238708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,1024,0.004152533411979675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,1024,0.015684266885121666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,5120,0.011464533209800721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,768,0.0038346665600935614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,768,0.015237333377202353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,4096,0.009833600123723347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,512,0.003587199995915095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,3584,0.009174399574597676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,512,0.015074132879575094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,3072,0.008184533317883809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,256,0.0032042667269706728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,256,0.014604799946149192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,128,0.0028661333024501802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,128,0.0145578662554423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,2560,0.007550933460394542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,64,0.002845866729815801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,64,0.014651733636856078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,32,0.0028917332490285236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,32,0.014596266547838846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,2048,0.006841599941253662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,65536,0.07901439666748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,65536,0.06286933422088622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,51200,0.06283626556396485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,1536,0.006487466891606649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,51200,0.051309867699941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,1024,0.005582933127880096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,16384,0.023548799753189086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,16384,0.025919999678929644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,768,0.005623466769854228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,12288,0.018845866123835243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,12288,0.02360853354136149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,512,0.005093333125114441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,10240,0.016169599692026772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,10240,0.022457599639892578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,256,0.004922666649023692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,128,0.004678399860858917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,8192,0.013691733280817667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,8192,0.021631999810536703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,7168,0.012599466244379678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,7168,0.021423999468485513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,65536,0.08024426301320395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,6144,0.011502933502197266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,6144,0.02068693240483602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,5120,0.010488532980283101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,51200,0.06473173300425211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,5120,0.019117865959803262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,4096,0.009290666381518046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,4096,0.017811199029286705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,16384,0.02309760053952535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,3584,0.008622933427492778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,3584,0.017130666971206666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,3584,0.008388266960779826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,3072,0.008024533092975617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,3072,0.01714986761411031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,12288,0.018441599607467652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,2560,0.0067456002036730455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,10240,0.01613866686820984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,2560,0.016636799772580466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,2048,0.005883733431498209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,2048,0.016254933675130208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,8192,0.013955199718475341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,1536,0.005026133358478546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,7168,0.012846933801968894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,1536,0.016586666305859886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,1024,0.0041802664597829185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,1024,0.01639573375384013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,6144,0.01169599990049998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,768,0.0038581334054470064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,768,0.015254400173823037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,5120,0.010668800274531046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,512,0.003576533248027166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,512,0.01487573285897573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,4096,0.00906880001227061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,256,0.003156266609827677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,256,0.014843733112017313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,3072,0.00767573316891988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,128,0.0028746667007605235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,128,0.014469333489735923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,128,0.004548266530036926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,64,0.002808533360560735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,64,0.014754133423169455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,32,0.002833066632350286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,32,0.01474133332570394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,65536,0.05288106600443522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,2560,0.007211733361085255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,65536,0.04944106737772624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,51200,0.04258879820505778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,51200,0.04172906478246053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,2048,0.006537599861621857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,1536,0.006073600053787232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,16384,0.018100267648696898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,16384,0.02335360050201416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,1024,0.005465599894523621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,12288,0.021385600169499717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,768,0.005335466563701629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,10240,0.012390399972597759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,10240,0.021149865786234536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,512,0.005063466727733612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,8192,0.0107232004404068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,8192,0.020364799102147422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,256,0.004805333415667216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,7168,0.010078932841618855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,12288,0.01383039951324463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,7168,0.01950506567955017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,6144,0.00907306671142578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,6144,0.01857066750526428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,65536,0.07742506663004557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,5120,0.00940053363641103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,5120,0.019025067488352455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,51200,0.061907200018564854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,4096,0.008497066299120585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,4096,0.017802667617797852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,12288,0.018310399850209554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,3584,0.007960533102353413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,3584,0.017824000120162962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,10240,0.015917866428693136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,3072,0.0068010667959849044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,3072,0.01713386575380961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,8192,0.01378666659196218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,2560,0.006354133288065593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,2560,0.01686613361040751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,16384,0.02283946673075358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,2048,0.005529599885145823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,7168,0.012322133779525757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,2048,0.01613653302192688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,6144,0.011079466342926026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,1536,0.00489386667807897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,1536,0.01581653356552124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,5120,0.009768533706665038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,1024,0.0042346666256586705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,1024,0.015609600146611533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,1024,0.0054506664474805195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,768,0.0038453333079814913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,4096,0.00870293378829956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,768,0.005070933202902476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,768,0.015093333522478738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,512,0.003553066651026408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,512,0.015075199802716575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,256,0.0031295999884605407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,256,0.015027200182278952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,3584,0.00826453318198522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,128,0.0028704000016053517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,128,0.014578133821487427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,3072,0.007625600198904674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,64,0.0027882667879263563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,64,0.014723199605941772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,32,0.0028351999819278715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,32,0.01471466620763143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,65536,0.04209920167922974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,2560,0.007276799778143566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,65536,0.04409493207931518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,51200,0.03436906735102336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,2048,0.006340266764163971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,51200,0.03779520193735759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,16384,0.01388053297996521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,16384,0.02227733333905538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,16384,0.02291839917500814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,12288,0.011801600456237793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,1536,0.005959466596444448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,12288,0.02143893241882324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,10240,0.010758399963378906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,10240,0.02077546715736389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,8192,0.01076479951540629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,8192,0.019042134284973145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,512,0.004973866542180379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,7168,0.01013866662979126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,256,0.00461760014295578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,7168,0.01894186735153198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,6144,0.009397332866986591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,6144,0.01856213410695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,128,0.004508799811204275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,5120,0.008358400066693623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,5120,0.018514132499694823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,65536,0.07766506671905518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,4096,0.007316266496976216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,4096,0.01777600049972534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,51200,0.06115200122197469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,3584,0.00694400022427241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,3584,0.017240534226099648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,10240,0.015847466389338174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,12288,0.018284799655278523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,3072,0.0063178668419520065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,3072,0.01665279964605967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,2560,0.006135466694831848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,8192,0.013141333063443502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,2560,0.0165173331896464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,2048,0.005388799806435903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,2048,0.016083199779192606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,7168,0.012026666601498922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,1536,0.004830933113892873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,1536,0.016260266304016113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,6144,0.010886399944623312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,1024,0.0041184000670909885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,1024,0.015428266922632852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,5120,0.00977280040582021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,768,0.0037791999677817024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,768,0.014932266871134438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,4096,0.008669867118199667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,512,0.003492266684770584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,512,0.015177599589029946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,3584,0.00809279978275299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,256,0.0030570665995279947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,256,0.014679466684659323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,3072,0.007575466732184092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,128,0.0029343999922275543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,128,0.014497066537539164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,2560,0.0071168000499407455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,64,0.002808533360560735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,64,0.014601600170135499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,32,0.0028266665836175283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,2048,0.006397866706053417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,32,0.014648532867431641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,65536,0.030624000231424968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,65536,0.03778453270594279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,51200,0.02534079949061076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,51200,0.03176533381144206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,1536,0.006002133091290792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,16384,0.010855467120806376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,16384,0.021530665953954062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,1024,0.005193600058555603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,12288,0.009691733121871948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,12288,0.019282132387161255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,768,0.005042133231957754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,512,0.00487253318230311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,10240,0.0100000003973643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,10240,0.018939733505249023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,128,0.004565333326657614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,8192,0.008772266904513042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,8192,0.01831679940223694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,65536,0.07729173501332601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,7168,0.008054399987061818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,51200,0.06143146753311157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,7168,0.011783466736475626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,6144,0.007525333265463512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,6144,0.01813439925511678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,256,0.004593066871166229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,5120,0.006748799979686737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,5120,0.01850559910138448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,16384,0.02271040081977844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,4096,0.006198399762312571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,4096,0.017808000246683754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,12288,0.01767680048942566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,3584,0.006683733562628429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,3584,0.01709866722424825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,3584,0.008165333171685536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,7168,0.018615466356277467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,3072,0.006211199859778086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,3072,0.016722132762273155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,2560,0.00610453337430954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,2560,0.016526933511098227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,2048,0.005458133419354757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,2048,0.01590293347835541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,10240,0.015092266599337259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,1536,0.004752000172932943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,1536,0.015889066457748412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,8192,0.012863999605178833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,1024,0.004062933226426443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,1024,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,6144,0.010666666428248088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,768,0.003718400001525879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,768,0.015280000368754067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,5120,0.009636267026265462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,512,0.003356799980004629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,512,0.014989866813023885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,4096,0.00848533312479655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,256,0.0030165334542592366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,256,0.01456106702486674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,3072,0.0075989335775375364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,128,0.0028597332537174227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,128,0.01455893317858378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,2560,0.007035733262697856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,64,0.002648533384005229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,64,0.014493866761525472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,32,0.002657066782315572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,2048,0.006415999929110209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,32,0.014536533753077188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,65536,0.01887786587079366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,65536,0.029028266668319702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,51200,0.015156267086664834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,51200,0.025592533747355144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,1536,0.006045866509278615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,16384,0.009086933732032777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,16384,0.01909760038057963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,1024,0.005273599922657013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,12288,0.00763733337322871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,12288,0.01853760083516439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,768,0.005044266581535339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,10240,0.007030400137106578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,10240,0.01867626706759135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,512,0.004681600133577982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,8192,0.006471466521422069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,8192,0.018129066626230875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,256,0.0046069333950678505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,7168,0.006263466676076253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,7168,0.01853546698888143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,128,0.00445119986931483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,6144,0.006022400160630544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,6144,0.01804479956626892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,6144,0.010758399963378906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,5120,0.006405333181222279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,5120,0.018228266636530557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,65536,0.07699840068817139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,4096,0.0060586666067441305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,4096,0.017324799299240114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,51200,0.061332265535990395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,16384,0.021795199314753214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,3584,0.017627733945846557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,12288,0.017237333456675212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,3072,0.00624533345301946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,3072,0.017026132345199584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,10240,0.014995200435320535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,2560,0.006053333481152853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,8192,0.012737066547075907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,2560,0.016448000073432924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,2048,0.0053962667783101406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,2048,0.01662399967511495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,7168,0.01179200013478597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,1536,0.004712533454100291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,1536,0.015549866358439126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,5120,0.009566932916641235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,3584,0.006516266862551372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,1024,0.0040618665516376495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,1024,0.015453867117563882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,4096,0.008680533369382222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,768,0.0036458666125933326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,3584,0.008196266492207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,768,0.014959999918937683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,512,0.003352533280849457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,512,0.01470186710357666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,3072,0.007570133109887441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,256,0.0030069333811601003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,256,0.014758400122324624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,256,0.00451093316078186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,128,0.0028170667588710784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,128,0.014528000354766845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,2560,0.007053866485754649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,64,0.002648533384005229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,64,0.014435199896494546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,32,0.002664533257484436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,2048,0.006422399977842967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,32,0.014467199643452963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,65536,0.011723732948303223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,65536,0.02564053336779277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,51200,0.010386133193969726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,51200,0.02295573353767395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,51200,0.06011626720428467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,16384,0.006215466558933258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,16384,0.01885653336842855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,1536,0.005915733178456625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,12288,0.00619946668545405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,12288,0.01887786587079366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,1024,0.0051925331354141235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,10240,0.00639573335647583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,10240,0.018563199043273925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,8192,0.0062389334042867025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,768,0.0051125332713127134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,8192,0.018175999323527016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,7168,0.006119466821352641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,7168,0.018615466356277467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,512,0.004783999919891357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,6144,0.005971199770768484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,6144,0.017926400899887084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,128,0.004409599800904592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,5120,0.006404266754786174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,5120,0.01832533280054728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,65536,0.07612799803415934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,4096,0.005994666616121928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,4096,0.017720532417297364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,4096,0.00860053300857544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,3584,0.006409599880377452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,16384,0.021663999557495116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,3584,0.017414400974909462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,3072,0.006118399898211161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,3584,0.008160000046094257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,3072,0.016682666540145875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,2560,0.005971199770768484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,12288,0.017296000321706136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,2560,0.01636693378289541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,2560,0.007035733262697856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,2048,0.005322666466236114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,2048,0.016327466567357382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,1536,0.004682666560014089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,1536,0.01567893326282501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,1536,0.006090666850407918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,1024,0.004043733328580856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,1024,0.015601066748301187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,10240,0.014934399724006652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,768,0.0036149332920710243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,768,0.015050666530927024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,8192,0.012865066528320312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,512,0.0034656000634034475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,512,0.01476800044377645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,7168,0.011781332890192668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,256,0.0030122667551040648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,256,0.014430933197339377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,6144,0.010776533683141073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,128,0.0028266665836175283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,128,0.014274133245150247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,5120,0.009646933277448018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,64,0.0026602665583292644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,64,0.01474240024884542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,32,0.0026496000587940215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,32,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,65536,0.009019733468691508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,65536,0.023778132597605386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,3072,0.007344000041484833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,51200,0.008008533219496409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,16384,0.006163200239340464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,51200,0.022261333465576173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,12288,0.005946666498978933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,16384,0.01909760038057963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,10240,0.006215466558933258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,12288,0.018447999159495035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,8192,0.00626453310251236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,2048,0.006313600142796834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,10240,0.01870186726252238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,8192,0.01776533325513204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,6144,0.0059114664793014525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,7168,0.01849173307418823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,5120,0.006287999947865804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,6144,0.01788160006205241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,4096,0.005841066439946493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,5120,0.018309332927068076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,4096,0.017366399367650352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,1024,0.005333333214124044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,3584,0.006326400240262349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,3072,0.005982933441797892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,3584,0.01758613387743632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,2560,0.005975466469923655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,2560,0.016474666198094685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,768,0.005050666630268097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,2048,0.0053727999329566956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,1536,0.004670933385690053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,2048,0.016173866391181946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,1024,0.003947733342647553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,512,0.0048991998036702475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,768,0.003643733263015747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,1024,0.015338666240374246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,512,0.003260799994071325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,768,0.015107199549674988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,256,0.0029887999097506206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,512,0.014936533570289613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,128,0.0027594665686289472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,256,0.014600533246994018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,256,0.004667733112970988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,64,0.002666666607062022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,128,0.014361600081125895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,64,0.014340266585350037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,32,0.0026752000053723653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,65536,0.009717333316802978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,32,0.014430933197339377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,65536,0.02402026653289795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,51200,0.008390399813652038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,16384,0.0061258668700853985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,51200,0.022436267137527464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,12288,0.006029866635799408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,16384,0.01925440033276876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,12288,0.01834986607233683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,10240,0.01858666737874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,8192,0.006066133578618368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,128,0.00453653335571289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,7168,0.006050133208433787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,8192,0.017834667364756265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,6144,0.005895466605822245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,5120,0.006270933151245117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,6144,0.01848213275273641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,7168,0.018158932526906334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,4096,0.005847466488679251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,5120,0.018148267269134523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,3584,0.0063285330931345625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,4096,0.01716266671816508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,3072,0.006005333364009857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,3584,0.017409066359202065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,3072,0.016709333658218382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,2560,0.005941333373387655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,3072,0.016548267006874083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,2048,0.005201066533724466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,2560,0.017244799931844076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,1536,0.0045962666471799215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,2048,0.01658453345298767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,1536,0.015364266435305276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,1024,0.003946666667858759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,768,0.0035743998984495797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,512,0.003270400067170461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,1024,0.015603199601173401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,768,0.014884266257286071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,1536,0.015594666202863058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,128,0.002717866748571396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,512,0.01495039959748586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,256,0.014471466342608133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,64,0.0026101333399613695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,32,0.002631466587384542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,128,0.014520532886187234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,64,0.014421332875887552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,32,0.014251733819643656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,7168,0.006137600044409434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,10240,0.006379733482996623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,16384,0.7515722910563152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,256,0.0029578665892283124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,16384,0.3969866752624512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,16384,0.3640970547993978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,10240,0.2540789286295573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,12288,0.27687145868937174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,12288,0.56953919728597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,10240,0.47780691782633467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,51200,1.189092254638672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,8192,0.24249067306518554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,8192,0.3827007929484049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,7168,0.18333120346069337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,7168,0.392305056254069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,51200,2.365461222330729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,6144,0.1624106725056966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,6144,0.287937068939209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,12288,0.3020384152730306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,5120,0.24027627309163413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,5120,0.13655254046122234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,5120,0.12040212949117024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,4096,0.11255786418914795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,4096,0.19397546450297037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,3584,0.1702730655670166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,3584,0.10800639788309734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,3072,0.14713493982950848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,3072,0.08935360113779703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,51200,1.1482986450195312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,10240,0.2324458599090576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,2560,0.12402026653289795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,8192,0.18881920178731282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,2560,0.07780160109202067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,2048,0.0997599999109904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,7168,0.17202986081441243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,2048,0.06573760112126668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,1536,0.07687786420186361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,1536,0.05450773239135742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,6144,0.16318400700887042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,4096,0.09905066490173339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,1024,0.06078399817148844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,1024,0.042344534397125246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,768,0.04192639986673991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,768,0.03649280071258545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,3584,0.08750720024108886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,768,0.02508266568183899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,512,0.02965013384819031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,512,0.029422932863235475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,256,0.017041067282358803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,256,0.022409600019454957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,128,0.010021332899729412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,128,0.019633066654205323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,128,0.011955199639002483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,3072,0.0858901341756185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,64,0.007463466624418895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,65536,32,0.0058442667126655575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,32,0.01793173352877299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,65536,64,0.01733120083808899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,2560,0.07265173594156901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,2048,0.05407253503799438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,1536,0.04204266468683879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,1024,0.030988800525665283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,512,0.019569067160288493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,65536,256,0.013940266768137612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,65536,1.195428212483724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,65536,1.1274495442708334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,51200,0.9336629231770834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,16384,0.5873642603556315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,16384,0.31413227717081704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,51200,1.8509877522786458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,65536,2.356817118326823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,12288,0.2689845403035482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,12288,0.4503434816996257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,51200,0.8784512201944986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,10240,0.3790229479471842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,10240,0.23149545987447104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,8192,0.29755093256632487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,8192,0.19001706441243488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,7168,0.26638933817545574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,7168,0.13300586541493734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,7168,0.1669098695119222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,6144,0.228111998240153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,6144,0.13294613361358643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,5120,0.11032746632893879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,5120,0.0955018679300944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,4096,0.15329705874125163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,4096,0.09196266333262125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,4096,0.07908373673756917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,3584,0.13653119405110675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,3584,0.08226133187611898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,3072,0.1155776023864746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,3072,0.07278506755828858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,5120,0.18788053194681803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,2560,0.09664533138275147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,2560,0.06360746622085571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,16384,0.3371530532836914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,2560,0.0525760014851888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,12288,0.2216543992360433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,2048,0.0892245372136434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,2048,0.05947306553522745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,1536,0.0602506677309672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,1536,0.04523733456929525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,1024,0.042497066656748454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,1536,0.03765439987182617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,1024,0.03572053511937459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,1024,0.025849600632985432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,768,0.032476800680160525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,768,0.030246400833129884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,768,0.021025067567825316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,10240,0.21257600784301758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,512,0.024913066625595094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,256,0.01304533382256826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,256,0.020130133628845213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,8192,0.17092053095499676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,6144,0.11338880062103271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,128,0.009192533294359843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,128,0.017705599466959633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,64,0.005025066435337067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,64,0.016658133268356322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,32,0.005085866649945577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,51200,32,0.016727467377980552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,2048,0.047729067007700604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,3584,0.0696821371714274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,51200,512,0.023204267024993896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,3072,0.061121066411336265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,65536,0.387061341603597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,65536,0.7460117340087891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,65536,0.36397120157877605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,51200,0.3049973487854004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,51200,0.5858432133992513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,16384,0.22060373624165855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,16384,0.11007040341695148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,12288,0.14191360473632814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,12288,0.0946453332901001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,12288,0.07653546333312988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,10240,0.11824746926625569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,10240,0.0744874636332194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,8192,0.09588800271352133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,512,0.01575040022532145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,8192,0.06334826548894247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,8192,0.05223360061645508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,7168,0.08423146406809488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,7168,0.057416534423828124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,256,0.0120170662800471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,6144,0.0518613338470459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,51200,128,0.008617599805196125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,5120,0.061954132715861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,5120,0.04559893210728963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,6144,0.07370560169219971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,4096,0.05021013418833414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,4096,0.03942933479944865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,51200,0.2883263905843099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,3584,0.04002879858016968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,3584,0.04437973499298096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,3072,0.038753068447113036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,16384,0.11347520351409912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,3072,0.03368213176727295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,3072,0.026616533597310383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,2560,0.03300586740175883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,2560,0.030848000446955365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,2560,0.02112320065498352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,2048,0.026997333765029906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,2048,0.027112533648808796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,1536,0.021237333615620933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,1536,0.023706666628519692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,10240,0.06367573340733847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,1024,0.016076800227165223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,1024,0.020822399854660036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,7168,0.04633066654205322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,768,0.011818666259447734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,768,0.019441066185633342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,6144,0.041018664836883545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,512,0.008965333302815754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,512,0.017941333850224814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,5120,0.03543999989827474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,256,0.006045866509278615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,256,0.0155349334081014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,256,0.006385066608587901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,128,0.003731200098991394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,128,0.015159466862678527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,128,0.0058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,64,0.003502933432658514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,64,0.015230933825174967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,16384,32,0.0036618667344252265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,16384,32,0.015430399775505066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,4096,0.02962239980697632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,3584,0.0269269327322642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,65536,0.3020394643147787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,65536,0.5622229258219401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,2048,0.017707733313242595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,65536,0.27833493550618493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,1536,0.014539733529090881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,16384,0.14809494018554686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,51200,0.4376202583312988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,51200,0.24360532760620118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,1024,0.011476266384124755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,12288,0.11412053108215332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,16384,0.09260266621907552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,12288,0.08094399770100912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,768,0.01016213297843933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,10240,0.062317868073781334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,10240,0.1112821340560913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,8192,0.07544533411661783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,8192,0.05433599948883057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,8192,0.04103573163350423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,7168,0.06586560010910034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,16384,512,0.008595200379689534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,7168,0.04895893335342407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,6144,0.05723520119984945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,5120,0.04770026604334514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,6144,0.044531198342641194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,5120,0.027999999125798543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,5120,0.038702932993570964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,4096,0.040934399763743086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,4096,0.03370346625645955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,4096,0.023670399188995363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,3584,0.03497813145319621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,3584,0.0327839990456899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,3584,0.02139520049095154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,3072,0.03126613299051921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,3072,0.029203200340270997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,2560,0.02571093241373698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,2560,0.02688639958699544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,16384,0.07584319909413656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,51200,0.2193450609842936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,12288,0.05779626766840616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,2048,0.024580266078313193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,2048,0.014261333147684732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,1536,0.0173632005850474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,2048,0.025988266865412397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,1536,0.022132267554601036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,1024,0.012345600128173827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,1024,0.019706666469573975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,10240,0.04972586631774902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,768,0.00999786655108134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,768,0.018844799200693766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,7168,0.03668906688690186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,768,0.008898133039474487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,512,0.007654400169849395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,512,0.015412267049153647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,6144,0.032232532898585006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,256,0.00407679999868075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,256,0.015378133455912272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,128,0.003533866753180822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,128,0.015046399831771851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,3072,0.019153066476186118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,64,0.0033546666304270422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,64,0.01511253317197164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,12288,32,0.0033781332274278007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,12288,32,0.01532266636689504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,65536,0.48895680109659834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,65536,0.2709312121073405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,2560,0.016777600844701132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,51200,0.38558613459269203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,1536,0.011754666765530903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,1024,0.009852799773216247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,16384,0.12856853008270264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,16384,0.0809610684712728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,51200,0.21027092933654784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,512,0.006911999980608622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,16384,0.06631146669387818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,12288,0.09882986545562744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,10240,0.08336319923400878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,12288,0.06351360082626342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,10240,0.05362559954325358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,8192,0.045873065789540604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,8192,0.06754986445109049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,128,0.005066666503747304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,7168,0.0545141339302063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,7168,0.04203626712163289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,7168,0.03174933393796285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,6144,0.03854933182398478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,6144,0.04788693189620972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,5120,0.040161065260569256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,5120,0.03437013228734334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,5120,0.024341332912445068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,4096,0.032892799377441405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,4096,0.03033493359883626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,12288,256,0.005655466516812643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,3584,0.03019839922587077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,3584,0.028878933191299437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,65536,0.2387221336364746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,3072,0.028895999987920123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,3072,0.026365866263707478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,3072,0.016707199811935424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,2560,0.02184106707572937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,51200,0.18778667449951172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,2560,0.025191466013590496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,2048,0.018026665846506754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,2048,0.02251840035120646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,10240,0.04412266810735067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,1536,0.014179199934005737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,1536,0.021259733041127524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,6144,0.02792746623357137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,1024,0.010353066523869832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,1024,0.01873813271522522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,4096,0.02074346741040548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,768,0.008686932921409606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,768,0.017233065764109292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,768,0.008278400202592214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,12288,0.04975786606470744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,512,0.006853333115577698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,512,0.015117866794268289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,256,0.0038592000802357995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,256,0.01508799990018209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,256,0.005342933535575867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,8192,0.03556053241093953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,128,0.003457066665093104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,128,0.015059199929237366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,64,0.0032245332996050516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,64,0.01504746675491333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,10240,32,0.003332266708215078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,3584,0.018564265966415406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,10240,32,0.014968533317248026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,65536,0.20930773417154946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,65536,0.3836288134256999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,65536,0.19629012743632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,2560,0.016429866353670754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,51200,0.3033226648966471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,2048,0.012688000003496805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,51200,0.16889813741048176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,16384,0.10986346403757732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,16384,0.06331520080566407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,12288,0.07168213526407877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,12288,0.0517525315284729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,1536,0.010616532961527507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,10240,0.06040639877319336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,10240,0.045337601502736406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,1024,0.009300266702969868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,8192,0.04903786579767863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,8192,0.03959253231684367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,512,0.006505600114663441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,7168,0.0434058666229248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,7168,0.03656426668167114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,10240,128,0.004990933338801066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,7168,0.027210666735966997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,6144,0.03824746608734131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,6144,0.03337706724802653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,5120,0.032229334115982056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,5120,0.030501333872477214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,51200,0.1560853322347005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,16384,0.055937067667643225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,4096,0.026498132944107057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,4096,0.02683839996655782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,3584,0.023643734057744344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,3584,0.025385600328445435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,12288,0.0417301336924235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,10240,0.03795733451843262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,3072,0.020794665813446044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,3072,0.023890133698781332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,2560,0.01791680057843526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,2560,0.022652800877889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,2560,0.013026133179664612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,2048,0.014755200346310934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,2048,0.021206400791803994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,1536,0.01162986656030019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,1536,0.01990293264389038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,8192,0.03033493359883626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,1024,0.008845866719881693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,1024,0.017514665921529136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,6144,0.024065067370732628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,768,0.007400533556938172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,768,0.015588266650835672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,5120,0.02108479936917623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,512,0.005886933207511902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,512,0.015399466951688132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,512,0.0066549330949783325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,256,0.0037098666032155357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,256,0.015343999862670899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,4096,0.01816640098889669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,3584,0.016338133811950685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,128,0.0033610666791598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,128,0.01476800044377645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,64,0.0030975999931494398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,64,0.01499626636505127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,3072,0.014888532956441245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,8192,32,0.003206400076548258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,2048,0.01162986656030019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,8192,32,0.01504746675491333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,1536,0.009912533561388652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,65536,0.3377141316731771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,65536,0.1975221316019694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,1024,0.008877866466840108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,65536,0.19428480466206868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,16384,0.0887829303741455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,16384,0.058101332187652587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,51200,0.1604522705078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,51200,0.2677557309468587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,12288,0.06486719846725464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,12288,0.04756906827290853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,10240,0.054731734593709315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,10240,0.04251093467076619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,8192,0.04349120060602824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,8192,0.037043201923370364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,8192,0.029183999697367353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,768,0.007538133362929027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,7168,0.03852800130844116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,7168,0.03401386737823486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,6144,0.03414719899495443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,6144,0.031752532720565795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,6144,0.022722133000691733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,256,0.005508266886075338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,5120,0.028858667612075804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,5120,0.028833067417144774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,4096,0.023893332481384276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,4096,0.025542400280634564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,8192,128,0.005269333223501841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,3584,0.02142080068588257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,3584,0.02478933334350586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,3584,0.015148799618085226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,3072,0.01882666746775309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,3072,0.023050665855407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,51200,0.15038612683614094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,16384,0.05551466544469198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,2560,0.018257067600886027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,2560,0.021949867407480873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,2048,0.013333333532015481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,12288,0.04296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,2048,0.020247467358907065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,1536,0.010843732953071594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,1536,0.018985599279403687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,10240,0.03826346794764201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,1024,0.008150400221347808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,7168,0.02603413263956706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,1024,0.015619200468063355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,1024,0.007481599847475688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,768,0.006975999971230824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,768,0.015568000078201295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,5120,0.020248534282048543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,512,0.0044501334428787235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,512,0.015102932850519816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,4096,0.016808533668518068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,256,0.0034656000634034475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,256,0.015271466970443726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,3072,0.013691733280817667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,128,0.0032437334458033243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,128,0.014822399616241455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,128,0.004877866804599762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,64,0.0031306666632493338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,64,0.014841600259145101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,7168,32,0.003143466760714849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,7168,32,0.014986667037010192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,2560,0.012341333429018657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,65536,0.2841130574544271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,2048,0.010912000139554342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,1536,0.009703466296195984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,65536,0.16493760744730632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,51200,0.2249301274617513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,51200,0.13331092993418375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,16384,0.07811733086903891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,768,0.007010133564472198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,16384,0.052184534072875974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,12288,0.06276799837748209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,12288,0.04463573296864827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,12288,0.03706666628519694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,10240,0.05285333395004273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,10240,0.03938026825586955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,8192,0.03573866685231526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,512,0.006263466676076253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,7168,0.03540800015131633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,7168,0.032529066006342575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,7168,256,0.005241600175698599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,7168,0.024546132485071818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,6144,0.03102933367093404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,6144,0.03020906647046407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,5120,0.02619733413060506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,5120,0.027477333943049114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,5120,0.019079466660817467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,4096,0.022292266289393105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,4096,0.024972800413767496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,16384,0.05441813468933106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,3584,0.019845332702000937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,3584,0.023833600680033366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,8192,0.04064106543858846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,3072,0.017526400089263917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,3072,0.022075732549031578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,65536,0.1771509329477946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,2560,0.01536853313446045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,51200,0.13979093233744305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,2560,0.02104319930076599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,2048,0.012339199582735699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,2048,0.019393066565195717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,10240,0.036374398072560626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,1536,0.010172800223032633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,1536,0.01874133348464966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,8192,0.027404799064000444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,1024,0.00772266685962677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,1024,0.01579839984575907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,6144,0.02148266633351644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,768,0.006595199803511302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,768,0.015470932920773825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,768,0.00695360004901886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,512,0.004095999896526337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,512,0.015386666854222616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,4096,0.016170666615168253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,256,0.0034442665676275887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,256,0.01502826710542043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,256,0.0051466668645540874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,128,0.003260799994071325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,128,0.014788267016410828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,3584,0.01434346636136373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,64,0.0030080000559488933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,64,0.014939733346303306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,6144,32,0.003078400095303853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,3072,0.013251200318336487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,6144,32,0.014920533696810404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,65536,0.1451690673828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,65536,0.24754239718119303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,51200,0.19734400113423664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,51200,0.11385066509246826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,51200,0.13291412989298504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,2560,0.012145066261291504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,16384,0.06517759958902994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,2048,0.010430933038393656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,16384,0.04998720089594523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,16384,0.048299733797709146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,12288,0.05083946784337362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,12288,0.03913386662801106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,10240,0.04323306481043498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,10240,0.03455573320388794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,8192,0.03543999989827474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,1536,0.009399466713269551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,8192,0.030508800347646074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,7168,0.028654932975769043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,7168,0.028629332780838013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,1024,0.007638399799664815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,6144,0.025752532482147216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,6144,0.026707200209299724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,512,0.0060597335298856105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,6144,128,0.004818133513132731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,5120,0.021966934204101562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,5120,0.024715733528137208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,5120,0.01821333368619283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,4096,0.01812373399734497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,4096,0.022789333264033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,65536,0.16863786379496257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,3584,0.01831679940223694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,3584,0.022139734029769896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,12288,0.03535573482513428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,10240,0.034671998023986815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,3072,0.02074880003929138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,8192,0.026178133487701417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,2560,0.012330666184425354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,2560,0.020087466637293497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,7168,0.023179733753204347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,2048,0.010627200206120808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,2048,0.018961066007614137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,6144,0.02036906679471334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,1536,0.008820266524950663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,1536,0.017554134130477905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,1536,0.008760533730189006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,1024,0.00691840002934138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,1024,0.015658666690190635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,3072,0.014295466740926108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,768,0.005385600030422211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,768,0.015307733416557312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,512,0.00386559988061587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,512,0.014937600493431092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,4096,0.015338666240374246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,256,0.0034122665723164878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,256,0.014944000045458474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,256,0.00517439991235733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,128,0.00317546675602595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,128,0.014726400375366211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,3584,0.013637333114941915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,64,0.002951466788848241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,64,0.014841600259145101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,5120,32,0.0030069333811601003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,5120,32,0.014922666549682616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,65536,0.19103147188822428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,3072,0.012893866499265036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,65536,0.11722986698150635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,51200,0.1516277313232422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,2048,0.010147200028101603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,51200,0.09534719785054525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,16384,0.05137493213017782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,16384,0.04172053337097168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,16384,0.04405759970347087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,12288,0.040336000919342044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,12288,0.03558506568272908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,1024,0.007642666498819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,768,0.006792533397674561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,10240,0.03465066750844319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,10240,0.030947200457255047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,10240,0.030282666285832722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,8192,0.028769065936406452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,8192,0.02752000093460083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,512,0.006062933305899302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,7168,0.02603306571642558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,7168,0.0260917325814565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,2560,0.011356799801190694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,6144,0.023078399896621703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,6144,0.024270933866500855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,5120,0.0180565337340037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,5120,0.022871466477711995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,5120,128,0.004739200075467428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,5120,0.016685867309570314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,4096,0.014889599879582724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,4096,0.02148053248723348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,4096,0.014138666788736978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,3584,0.01341973344484965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,3584,0.020504534244537354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,3072,0.011959466338157653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,3072,0.020039467016855876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,65536,0.14921387036641437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,2560,0.010518399874369304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,2560,0.01941759983698527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,2560,0.010599467158317565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,2048,0.009012266993522644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,2048,0.018198400735855103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,51200,0.12036800384521484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,1536,0.007725866635640462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,1536,0.01614400049050649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,1536,0.008082133531570435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,1024,0.006211199859778086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,1024,0.015599999825159708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,12288,0.03149973352750142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,768,0.0043722664316495265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,768,0.015531733632087708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,768,0.00690773328145345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,512,0.003719466676314672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,512,0.015256533026695251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,8192,0.02396906614303589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,256,0.003291733314593633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,7168,0.021706666549046835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,256,0.014877866705258688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,128,0.003031466652949651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,6144,0.019778132438659668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,128,0.01469546655813853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,128,0.0047647997736930845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,64,0.0029365333418051405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,64,0.014684800306955972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,4096,32,0.0029845332105954488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,4096,32,0.014582399527231851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,3584,0.01318186620871226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,3072,0.011945600310961407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,65536,0.17385813395182292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,65536,0.10948800245920817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,51200,0.13752106030782063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,51200,0.08949546813964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,16384,0.04601066509882609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,16384,0.038712533315022786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,2048,0.009417600433031718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,12288,0.03634346723556518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,12288,0.03334720134735107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,12288,0.029398399591445922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,10240,0.031189332405726116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,1024,0.007470933099587758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,10240,0.02924586733182271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,8192,0.026039467255274458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,8192,0.02581760088602702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,512,0.006020266811052958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,7168,0.023577600717544556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,7168,0.02469653288523356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,7168,0.020078933238983153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,4096,256,0.005171200136343638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,6144,0.02363199989000956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,5120,0.016301866372426352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,5120,0.02215893268585205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,65536,0.1342378616333008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,4096,0.014443733294804893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,4096,0.020600533485412596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,51200,0.10607466697692872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,3584,0.013838932911554972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,3584,0.019962666432062785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,16384,0.04103253285090129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,3072,0.010980266332626342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,3072,0.019438934326171876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,6144,0.019203199942906698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,2560,0.009737599889437358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,2560,0.019091200828552247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,10240,0.028868265946706134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,2048,0.008373333017031352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,2048,0.016673066218694053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,4096,0.013356799880663553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,8192,0.02235413392384847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,1536,0.007342933118343354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,1536,0.016250666975975037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,1024,0.0050784001747767125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,1024,0.015457066893577575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,6144,0.01833066741625468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,768,0.004006399959325791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,768,0.01560640037059784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,5120,0.015946666399637856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,512,0.0036778666079044344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,512,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,3584,0.012361600001653036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,256,0.0032799998919169106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,3072,0.011455999811490376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,256,0.014858667055765787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,128,0.0030037333567937215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,128,0.014598400394121806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,2560,0.010326400399208069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,64,0.002881066749493281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,64,0.014824533462524414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3584,32,0.0029994666576385496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3584,32,0.014729600151379904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,65536,0.14772799809773762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,65536,0.09609920183817545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,2048,0.008566400408744812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,51200,0.11743040084838867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,1536,0.008257066706816356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,1024,0.007107200225194295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,16384,0.04205226500829061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,51200,0.12128640015920003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,16384,0.03570559819539388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,12288,0.03251840074857076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,12288,0.030929066737492877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,12288,0.031549866994222006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,10240,0.02792853315671285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,10240,0.02826346755027771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,768,0.006384000182151794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,8192,0.024180267254511514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,8192,0.026246400674184163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,512,0.005842133363087972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,7168,0.021669334173202513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,7168,0.024486400683720908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,51200,0.07771733601888021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,6144,0.019266132513682047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,6144,0.022348799308141074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,256,0.00514026681582133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,5120,0.01678826610247294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3584,128,0.0047882666190465295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,5120,0.021345067024230956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,5120,0.01650773286819458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,4096,0.014387200276056925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,4096,0.020398932695388793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,4096,0.014151466886202493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,3584,0.013217066725095114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,3584,0.020062933365503945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,3584,0.012560000022252401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,3072,0.010305066903432209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,3072,0.01909653345743815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,65536,0.15338026682535807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,2560,0.009092266360918682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,2560,0.01714986761411031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,16384,0.04526079893112182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,2048,0.008027733365694682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,2048,0.01634239951769511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,2048,0.008724266290664673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,1536,0.006820266445477803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,1536,0.0158869336048762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,10240,0.031557333469390866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,1024,0.004614399870236715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,1024,0.015844266613324484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,1024,0.00718506673971812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,768,0.00417493333419164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,768,0.015242666999499003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,8192,0.024152533213297526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,512,0.0035904000202814737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,512,0.015229866902033488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,7168,0.021228800217310585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,256,0.0033013333876927694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,256,0.014711466431617738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,6144,0.018639999628067016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,256,0.005022933085759481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,128,0.0029333333174387617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,128,0.014523733655611673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,64,0.0031136001149813337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,64,0.01453013320763906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,3072,32,0.0032085334261258446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,3072,32,0.014708266655604044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,65536,0.12599573135375977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,65536,0.08360640207926431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,51200,0.1010858694712321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,3072,0.01183573305606842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,51200,0.066920534769694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,16384,0.03456533352533976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,16384,0.03255573312441508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,2560,0.010009599725405376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,12288,0.027537065744400024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,12288,0.027845333019892376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,12288,0.028884265820185346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,10240,0.02404586672782898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,1536,0.008130133152008057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,10240,0.025883734226226807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,8192,0.020131200551986694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,8192,0.02387626568476359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,8192,0.021681066354115805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,7168,0.01816213329633077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,7168,0.02290880084037781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,768,0.006607999900976817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,6144,0.015987199544906617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,6144,0.022188800573349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,6144,0.017004799842834473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,5120,0.014062933127085366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,5120,0.02063680092493693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,5120,0.015196800231933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,4096,0.012059733271598816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,512,0.005846400062243144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,4096,0.019466666380564372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,3584,0.011356799801190694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,3584,0.01992853283882141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,3072,128,0.004602666695912679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,3072,0.009257599711418152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,3072,0.01845759948094686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,65536,0.1373045285542806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,2560,0.008462933699289958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,51200,0.11017173131306965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,2560,0.017485866943995156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,2560,0.009083732962608337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,2048,0.007284266750017802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,2048,0.016619732975959776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,1536,0.005966933568318685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,1536,0.0160671999057134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,16384,0.04114240010579427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,1024,0.004609066744645437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,1024,0.01567359964052836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,10240,0.027729066212972005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,768,0.004062933226426443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,768,0.015477333466211954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,7168,0.01939520041147868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,512,0.0036458666125933326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,512,0.01495680014292399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,4096,0.013103999694188437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,256,0.003294933338960012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,3584,0.01158186693986257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,256,0.014811733365058899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,128,0.0029685333371162414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,128,0.014775466918945313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,3072,0.010546132922172546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,64,0.0028234665592511495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,64,0.014632532993952433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2560,32,0.0028437333802382152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2560,32,0.014653866489728292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,65536,0.10019839604695638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,65536,0.06917119820912679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,65536,0.13264106909434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,2048,0.00803413341442744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,51200,0.07775999704996744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,1536,0.007286400099595388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,51200,0.05752000013987223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,1024,0.00695253312587738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,16384,0.028756266832351683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,16384,0.029217066367467244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,16384,0.03988906542460124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,12288,0.023014400402704874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,12288,0.025460267066955568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,10240,0.01999146739641825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,10240,0.023873066902160643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,768,0.006145066519578298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,8192,0.016396799683570863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,8192,0.02214720050493876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,512,0.00561706672112147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,7168,0.015005866686503092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,7168,0.0213045338789622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,256,0.0050346667567888895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,6144,0.014545067151387533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,6144,0.02099519968032837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,5120,0.01206826666990916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,5120,0.0200437327226003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,5120,0.0150218665599823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,4096,0.010456533233324686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,4096,0.01927466591199239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2560,128,0.004657066861788432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,51200,0.10498987038930256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,3584,0.009860266248385112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,3584,0.01773653427759806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,3072,0.00843946635723114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,3072,0.017141334215799966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,12288,0.028100266059239702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,2560,0.007749333480993907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,2560,0.016293332974116007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,2560,0.009147733449935913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,2048,0.006911999980608622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,10240,0.027151999870936076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,2048,0.016201600432395935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,1536,0.004822400212287903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,1536,0.01575146714846293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,1536,0.007507200042406718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,1024,0.004218666752179464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,8192,0.021948800484339396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,1024,0.015317333738009134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,1024,0.006325333317120869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,768,0.0038762666285037995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,768,0.01524906655152639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,512,0.003454933315515518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,512,0.015106133619944253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,7168,0.019099734226862588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,256,0.00317546675602595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,256,0.014641066392262777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,256,0.005060266455014547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,128,0.0029279999434947968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,128,0.014612266421318054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,6144,0.017232000827789307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,64,0.00296426663796107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,64,0.014662399888038635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,2048,32,0.003011200080315272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,2048,32,0.014647466937700906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,65536,0.07801493008931479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,65536,0.05937813520431519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,512,0.005830400188763936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,51200,0.062122666835784913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,51200,0.04903680086135864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,4096,0.012758400042851767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,16384,0.023421865701675416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,16384,0.025939200321833295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,3584,0.011013333002726238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,12288,0.018807466824849448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,3072,0.010284800330797832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,12288,0.023462400833765665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,12288,0.024077866474787393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,10240,0.016184533635775246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,10240,0.024793599049250284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,10240,0.022286933660507203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,8192,0.013612799843152366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,8192,0.020894932746887206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,2048,0.008289066453774769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,7168,0.012668800354003907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,7168,0.020193066199620566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,6144,0.011556266744931539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,6144,0.01990506649017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,6144,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,5120,0.010428800185521444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,5120,0.01928106745084127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,768,0.006647466619809468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,4096,0.009341866771380106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,4096,0.017509333292643228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,4096,0.010390399893124899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,3584,0.008870399991671244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,3584,0.01750719944636027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,2048,128,0.004721066852410635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,3072,0.008386133114496867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,3072,0.017205333709716795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,65536,0.11275413036346435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,2560,0.007339733342329661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,2560,0.01644159952799479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,51200,0.08807786305745444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,16384,0.03450133403142293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,2048,0.005824000140031179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,2048,0.01622719963391622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,8192,0.018531199296315512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,1536,0.00480320006608963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,7168,0.016709333658218382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,1536,0.015904000401496886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,1024,0.004099199920892716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,1024,0.015528532862663268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,5120,0.01304213305314382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,768,0.003852800031503042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,768,0.014983466267585755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,768,0.005930666625499725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,512,0.00342399999499321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,512,0.014904533823331198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,3584,0.009586133559544881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,256,0.0032981333633263906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,256,0.01471573313077291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,3072,0.008936533331871032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,128,0.0029578665892283124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,128,0.014567466576894126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,2560,0.00804906686147054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,64,0.0028575999041398365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,64,0.014493866761525472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1536,32,0.002937600016593933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1536,32,0.014556800325711569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,65536,0.05240426858266195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,65536,0.04626773198445638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,51200,0.04209493398666382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,65536,0.09136640230814616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,2048,0.007321600119272869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,51200,0.039239466190338135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,16384,0.016871466239293417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,16384,0.022707200050354003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,12288,0.013890133301417033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,12288,0.020988800128300986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,1536,0.006843733290831248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,10240,0.012473600109418233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,1024,0.006356266637643178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,10240,0.020850133895874024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,8192,0.010684800148010255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,8192,0.020268799861272176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,512,0.005354666709899902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,7168,0.010158933202425639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,256,0.004913066824277243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,7168,0.018733867009480796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,6144,0.00906773308912913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,6144,0.018067200978597008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1536,128,0.004598399996757508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,5120,0.009084799885749817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,5120,0.018785067399342856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,5120,0.009956266482671101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,4096,0.008170666793982189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,4096,0.018313600619633993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,51200,0.07139840126037597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,16384,0.028614399830500285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,3584,0.007413333157698314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,3584,0.01825173298517863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,3072,0.006838400165239971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,3072,0.017198934157689413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,3072,0.0076106667518615724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,2560,0.00681386689345042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,2560,0.016691199938456216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,10240,0.021670399109522502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,2048,0.005932799975077311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,8192,0.015562666455904641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,2048,0.01665279964605967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,1536,0.004949333270390829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,1536,0.015927466750144958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,1536,0.006145066519578298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,1024,0.004355200131734212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,1024,0.01528320014476776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,7168,0.013380266229311624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,768,0.003985066711902618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,768,0.015247999628384908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,6144,0.011170132954915365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,512,0.003570133447647095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,512,0.014916266997655234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,12288,0.01973973313967387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,512,0.004977066814899444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,256,0.003127466638882955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,256,0.014535466829935709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,128,0.0028714666763941447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,128,0.01440000037352244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,4096,0.008711466193199157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,64,0.0028213332096735638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,64,0.01442026694615682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,1024,32,0.0028192001084486645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,1024,32,0.01458453337351481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,65536,0.0414741317431132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,65536,0.040601599216461184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,3584,0.008221866687138875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,51200,0.03385066588719686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,51200,0.03483839829762776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,2560,0.0071274667978286745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,16384,0.013920000195503235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,2048,0.0065536002318064375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,16384,0.021809067328770956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,12288,0.011848533153533935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,12288,0.02001706759134928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,1024,0.00581226646900177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,768,0.005257600049177805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,10240,0.010825600226720173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,10240,0.019192532698313395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,256,0.004834133386611939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,8192,0.009459199508031209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,8192,0.01775040030479431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,1024,128,0.004525866607824961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,7168,0.008876799543698629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,7168,0.020603734254837035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,65536,0.0782602628072103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,6144,0.008227199812730153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,6144,0.01899519960085551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,51200,0.06500693162282309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,5120,0.007438933352629344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,5120,0.018668800592422485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,5120,0.009883733590443929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,16384,0.02403200070063273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,4096,0.0065194666385650635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,4096,0.01768959959348043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,3584,0.007805866499741872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,3584,0.017385600010553996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,3072,0.006832000116507213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,12288,0.01790293256441752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,3072,0.016769067446390788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,3072,0.007480533421039581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,2560,0.006129066646099091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,2560,0.016210132837295534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,2048,0.005438933273156484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,2048,0.016479999820391337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,10240,0.01767573356628418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,1536,0.004802133142948151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,1536,0.015646933515866598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,8192,0.013063466548919678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,1024,0.004142933338880539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,1024,0.015190399686495461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,7168,0.012154666582743327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,768,0.0037578667203585305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,768,0.014958932995796204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,6144,0.010598400235176086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,512,0.0034261333445707956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,512,0.014914133151372275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,4096,0.008438400427500407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,256,0.003083733220895131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,256,0.014656000336011252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,3584,0.008090666433175405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,256,0.004646400113900503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,128,0.0028970666229724885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,128,0.014389333128929139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,128,0.004479999840259552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,64,0.0027114666998386385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,768,32,0.002776533365249634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,64,0.014641066392262777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,65536,0.029843199253082275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,768,32,0.014441600441932679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,2560,0.007119999825954437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,51200,0.024637866020202636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,2048,0.006303999821345012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,1536,0.0059125334024429325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,16384,0.010986666878064473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,16384,0.020053333044052123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,1024,0.005543466905752818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,12288,0.009726933638254802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,12288,0.018229333559672038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,768,0.005081599950790406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,12288,0.01717546582221985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,10240,0.00871573289235433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,10240,0.014914133151372275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,10240,0.019662932554880778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,8192,0.007805866499741872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,8192,0.018694400787353516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,65536,0.03440106709798177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,7168,0.007180800040562947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,7168,0.01857173244158427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,768,512,0.00487253318230311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,51200,0.029268266757329305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,6144,0.006537599861621857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,6144,0.018001067638397216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,5120,0.018245333433151247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,5120,0.008116266628106435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,4096,0.007000533243020375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,65536,0.07489706675211588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,4096,0.01763733426729838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,4096,0.008467200398445129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,3584,0.006657066444555919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,3584,0.008077866832415263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,3584,0.017254400253295898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,3072,0.006260266900062561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,3072,0.016511999567349753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,3072,0.007470933099587758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,2560,0.006031999985376993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,2560,0.016499200463294984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,2048,0.005448533097902933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,2048,0.016268799702326454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,16384,0.02243946592013041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,1536,0.004705066482226053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,1536,0.015761066476504007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,8192,0.01267413298288981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,1024,0.004012800008058548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,1024,0.015307733416557312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,6144,0.01067200005054474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,51200,0.060426668326059974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,768,0.003638399889071783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,5120,0.009581866860389709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,768,0.015316266814867655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,768,0.005031466484069824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,512,0.0033312000334262846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,512,0.014798933267593383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,256,0.003045333425203959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,256,0.014619732896486918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,7168,0.011848533153533935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,128,0.0029418667157491045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,128,0.0145087997118632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,128,0.004625066618124644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,64,0.00275093341867129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,64,0.014572800199190775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,512,32,0.0027615999182065325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,512,32,0.014481066664059957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,65536,0.017780266205469766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,65536,0.026360533634821576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,2560,0.0070698668559392285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,51200,0.014281599720319112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,51200,0.023961599667867026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,2048,0.00629013329744339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,16384,0.008138666550318401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,16384,0.019053866465886436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,1536,0.005782400071620941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,12288,0.007605333129564922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,12288,0.01837973395983378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,1024,0.00517439991235733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,10240,0.00692799985408783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,10240,0.018355200688044228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,10240,0.014800000190734863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,8192,0.006436266501744588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,8192,0.01793173352877299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,7168,0.006479999919732411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,512,0.0047775998711586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,7168,0.01829973260561625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,6144,0.006109866499900818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,512,256,0.004589866598447164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,6144,0.017986132701237997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,5120,0.0064735998709996535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,5120,0.018294399976730345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,65536,0.07527360121409098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,4096,0.006107733150323232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,4096,0.01789120038350423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,51200,0.05990720192591349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,3584,0.006444799900054932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,3584,0.0168938676516215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,16384,0.021385600169499717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,3072,0.006172800064086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,3072,0.016862932840983072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,3072,0.007429333527882893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,2560,0.006106666723887126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,2560,0.0165503998597463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,12288,0.016938666502634682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,2048,0.005363200108210246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,2048,0.016157866517702738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,2048,0.006259199976921081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,1536,0.0046858668327331545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,1536,0.015615999698638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,8192,0.012643200159072877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,1024,0.004051200052102407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,7168,0.011709866921106975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,1024,0.015041066209475198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,768,0.003714133302370707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,6144,0.010636799534161885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,768,0.014937600493431092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,512,0.0034133332471052804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,5120,0.009499733646710713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,512,0.014792533715566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,256,0.0030389333764712016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,256,0.01460693379243215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,4096,0.008551466464996337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,768,0.0049098665515581764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,128,0.002922666569550832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,128,0.01444586714108785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,64,0.0026101333399613695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,3584,0.008014933268229166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,64,0.014533332983652749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,256,32,0.00262719988822937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,256,32,0.014498133460680643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,65536,0.011949867010116577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,65536,0.02360960046450297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,51200,0.010612266262372334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,51200,0.022318933407465616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,51200,0.059322667121887204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,2560,0.007028266787528992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,16384,0.006872533261775971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,16384,0.01890453298886617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,12288,0.006071466704209646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,12288,0.01832853356997172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,1536,0.005791999896367391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,10240,0.006333866715431213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,10240,0.01869759956995646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,1024,0.00522986650466919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,8192,0.006316799918810527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,8192,0.017836799224217735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,512,0.004802133142948151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,7168,0.006182399888833364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,256,0.004529066880544027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,7168,0.018092799186706542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,256,128,0.0044714664419492085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,6144,0.006010666489601135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,6144,0.017760000626246133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,6144,0.010708266496658325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,5120,0.006384000182151794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,65536,0.07413012981414795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,5120,0.018246400356292724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,4096,0.005966933568318685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,4096,0.017590399583180746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,16384,0.021473066012064616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,3584,0.0063733334342638654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,3584,0.017267199357350667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,12288,0.01692906618118286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,3072,0.006073600053787232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,3072,0.017079466581344606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,10240,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,2560,0.006046933432420095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,2560,0.016359466314315795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,8192,0.01262079974015554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,2048,0.0053258667389551794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,2048,0.016160000363985697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,2048,0.006248533229033152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,1536,0.0046623999873797095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,1536,0.005820799867312113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,1536,0.015808000167210897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,1024,0.004082133372624716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,1024,0.015352533260981242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,7168,0.011739733815193176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,768,0.0036874666810035707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,768,0.014961066842079162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,512,0.003328000009059906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,512,0.01479039986928304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,512,0.0047647997736930845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,256,0.0030250666042168934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,256,0.014443733294804893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,5120,0.00946666697661082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,128,0.0028522667785485585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,128,0.014413866400718688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,4096,0.008451199531555176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,64,0.00264533335963885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,64,0.01448853313922882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,128,32,0.0027146667242050173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,128,32,0.014450132846832275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,65536,0.009258666634559631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,3584,0.008054399987061818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,51200,0.008227199812730153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,65536,0.02188906669616699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,3072,0.007344000041484833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,16384,0.006083199878533682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,12288,0.006067200005054474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,16384,0.018935465812683107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,10240,0.006300800045331319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,12288,0.01819093426068624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,8192,0.006089599927266439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,2560,0.007010133564472198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,10240,0.017989333470662436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,7168,0.005955199897289276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,7168,0.01851946711540222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,6144,0.0058335999647776285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,5120,0.006209066510200501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,6144,0.017861332496007284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,4096,0.005835733314355215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,5120,0.01845546762148539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,3584,0.006297599772612255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,4096,0.017388800779978432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,3072,0.005916800101598104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,3584,0.017214934031168617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,2560,0.0060362666845321655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,3072,0.016633599996566772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,2048,0.005312000215053558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,2560,0.016379732886950174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,1536,0.004678399860858917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,2048,0.015972266594568886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,1024,0.004006399959325791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,1536,0.015824000040690102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,768,0.00365226666132609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,1024,0.015195733308792115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,512,0.003257599969704946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,768,0.015054933230082192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,1024,0.0052821333209673565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,256,0.002985599885384242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,512,0.014878933628400167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,128,0.0027530667682488757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,256,0.014651733636856078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,64,0.0026378666361172995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,128,0.01437013347943624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,64,32,0.0026880001028378804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,768,0.004970666766166687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,64,0.01432960033416748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,32,0.014330666263898215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,65536,0.010359467069307963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,51200,0.00874773363272349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,16384,0.0060693333546320595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,65536,0.022296533981959025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,51200,0.022580265998840332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,16384,0.018410666783650716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,10240,0.0061471998691558834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,12288,0.01810986598332723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,8192,0.0061482667922973635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,51200,0.020606933037439983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,10240,0.01835093299547831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,7168,0.005941333373387655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,8192,0.017571200927098594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,7168,0.017809067169825235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,6144,0.0058335999647776285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,5120,0.006141866743564606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,6144,0.01776319940884908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,5120,0.01807360053062439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,4096,0.005729066828886667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,256,0.004686933259169261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,3584,0.006278400123119354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,3072,0.0059232001503308615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,4096,0.017514665921529136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,3584,0.016781866550445557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,2560,0.00600853314002355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,3072,0.01687893271446228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,2048,0.005348266661167144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,2560,0.016532267133394875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,1536,0.004634666442871094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,2048,0.01604480048020681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,1024,0.003928533444801966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,1536,0.015465600291887918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,768,0.0036298667391141256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,1024,0.015415466825167336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,512,0.003349333256483078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,768,0.015197867155075073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,512,0.0147189329067866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,128,0.0027776000400384264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,256,0.014359466234842935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,64,0.0026517334083716077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,128,0.014301866292953491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,32,0.0026858667532602947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,64,0.014284800489743552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,32,32,0.014318933089574179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,33,128,128,0.004428799947102865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,12288,0.006078933179378509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,33,64,8192,0.01759786605834961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,33,32,256,0.002935466667016347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,16384,0.39594666163126624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,16384,0.7495701471964519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,16384,0.36500374476114905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,12288,0.30074453353881836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,12288,0.5689119974772135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,51200,1.1875935872395833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,10240,0.47629439036051435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,10240,0.2532533327738444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,10240,0.23185067176818847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,8192,0.4484896024068196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,8192,0.20721066792805992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,7168,0.18305919965108236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,51200,2.34432856241862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,7168,0.33259414037068685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,7168,0.17165546417236327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,6144,0.1592682679494222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,6144,0.28693227767944335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,5120,0.23947839736938475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,5120,0.1398527940114339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,4096,0.1935167948404948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,4096,0.11190933386484783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,3584,0.10030186971028646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,3584,0.16961174011230468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,3072,0.14643947283426922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,3072,0.08886079788208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,12288,0.2844586690266927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,51200,1.1453931172688803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,2560,0.14338773091634113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,2560,0.08714026610056559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,2048,0.11504106521606446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,2048,0.06533120075861612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,8192,0.1888576030731201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,1536,0.08838613033294677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,1536,0.053643735249837246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,6144,0.1631157398223877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,1536,0.0421887993812561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,5120,0.12066453297932941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,1024,0.04120853344599406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,1024,0.06040639877319336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,1024,0.034227200349171955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,768,0.041333333651224775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,4096,0.09870186646779379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,512,0.028970666726430255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,768,0.03880746761957805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,512,0.028292266527811687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,256,0.018607999881108603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,256,0.02140053311983744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,128,0.009491200248400371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,128,0.018888533115386963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,3584,0.08709759712219238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,128,0.011814399560292562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,64,0.007468800246715546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,32,0.0053269331653912864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,64,0.01709866722424825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,32,0.01725013256072998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,3072,0.07636480331420899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,2560,0.06696213086446126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,2048,0.05386879841486612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,65536,1.1944544474283854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,768,0.02512960036595662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,512,0.019291732708613077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,256,0.01404159963130951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,65536,2.349016571044922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,16384,0.5900618871053059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,51200,1.8474068959554035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,51200,0.9444192250569662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,16384,0.31503254572550454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,16384,0.33749866485595703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,51200,0.8775146484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,12288,0.4410655975341797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,10240,0.23038293520609537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,12288,0.2754272143046061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,10240,0.3705418586730957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,8192,0.29601173400878905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,8192,0.16537173589070636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,7168,0.2585450649261475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,8192,0.17037866910298666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,7168,0.14946773846944172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,7168,0.13008853594462078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,6144,0.2255626678466797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,6144,0.1287274678548177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,6144,0.1129151980082194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,5120,0.19606827100118002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,4096,0.1533258597056071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,5120,0.11482346852620443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,4096,0.10183786551157634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,3584,0.13214826583862305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,65536,1.124836222330729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,3584,0.08221333026885987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,3072,0.07229973475138346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,3072,0.13266346454620362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,2560,0.09611946741739909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,3072,0.060610131422678626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,2560,0.06970773537953695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,2048,0.07931199868520102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,2048,0.04303786754608154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,1536,0.06004053354263306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,1536,0.044973866144816084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,12288,0.25303680102030435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,10240,0.2117098649342855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,1024,0.04725013176600139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,1024,0.0378549337387085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,2048,0.05674026807149252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,768,0.036218667030334474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,768,0.03248213330904643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,512,0.02302186687787374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,512,0.02477226654688517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,512,0.015358933806419372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,256,0.012386133273442585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,256,0.019723733266194664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,5120,0.09537279605865479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,128,0.00881706674893697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,4096,0.07899626890818277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,128,0.017166932423909508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,64,0.004637866715590159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,64,0.016618667046229045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,51200,32,0.004802133142948151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,3584,0.06965760389963785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,51200,32,0.016809600591659545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,2560,0.052604798475901285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,65536,0.3838794708251953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,1536,0.034074668089548746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,65536,0.7431349436442057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,65536,0.363262939453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,16384,0.18660373687744142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,51200,0.3046698570251465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,51200,0.5840672175089519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,51200,0.29410133361816404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,16384,0.10969173113505046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,16384,0.10358933607737224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,12288,0.098471466700236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,12288,0.16669333775838216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,1024,0.025829333066940307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,10240,0.11777493158976238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,10240,0.07487680117289225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,8192,0.09649600187937418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,8192,0.06333333253860474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,768,0.020694400866826376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,7168,0.08432640234629313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,7168,0.0572703997294108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,6144,0.0735040028889974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,6144,0.051413333415985106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,6144,0.04102186759312947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,5120,0.06116480032602946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,256,0.011958400408426922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,4096,0.050054399172465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,4096,0.03918399810791016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,51200,128,0.009060266613960265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,4096,0.02947946588198344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,3584,0.044251732031504315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,3584,0.03663359880447388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,3584,0.026891734202702838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,3072,0.03876800139745076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,3072,0.033557331562042235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,2560,0.03287893335024516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,2560,0.030565333366394044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,5120,0.04535679817199707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,2048,0.02690773407618205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,2048,0.02717653314272563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,12288,0.07451626459757486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,10240,0.06517653465270996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,8192,0.05929493506749471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,1536,0.021166932582855225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,1536,0.024001065889994302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,1024,0.014923733472824097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,1024,0.020807466904322305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,5120,0.035327998797098796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,768,0.011783466736475626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,768,0.01944640080134074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,3072,0.02400533358256022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,512,0.00876586635907491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,512,0.01819733381271362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,512,0.00853653351465861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,256,0.005998933315277099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,2560,0.021104000012079873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,256,0.01546986699104309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,128,0.0036864000062147772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,128,0.015174399813016257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,7168,0.04666879971822103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,64,0.0034400001168251038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,64,0.015209600329399109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,32,0.0035360001027584078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,32,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,2048,0.01768959959348043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,65536,0.5704853057861328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,65536,0.30548159281412757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,65536,0.2773077328999837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,1536,0.014780799547831217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,51200,0.44045438766479494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,51200,0.2410879929860433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,1024,0.011533866326014202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,16384,0.14652160008748372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,16384,0.0752458651860555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,768,0.010172800223032633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,12288,0.1133461316426595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,16384,0.102729598681132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,12288,0.05757333437601725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,12288,0.07239253520965576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,51200,0.21792640686035156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,10240,0.09549012978871664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,10240,0.06313386758168539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,8192,0.07806293169657388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,8192,0.05370239814122518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,7168,0.048767999807993574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,7168,0.06804479757944742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,6144,0.059000531832377114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,6144,0.04415359894434611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,5120,0.049075198173522946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,5120,0.03891306718190511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,4096,0.04070293505986532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,4096,0.034414935111999514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,4096,0.023833600680033366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,256,0.006439466774463653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,3584,0.03577280044555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,3584,0.021346133947372437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,3584,0.03156053423881531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,3072,0.0312394658724467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,3072,0.02890666723251343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,3072,0.01914986570676168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,2560,0.025574400027592974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,2560,0.026665600140889485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,2560,0.0167797327041626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,2048,0.021817600727081297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,2048,0.023780266443888344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,1536,0.01673706571261088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,1536,0.021699200073877968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,10240,0.049353599548339844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,1024,0.013513599832852682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,1024,0.01939093271891276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,6144,0.0321941335995992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,1024,0.009714133540789286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,768,0.009858133395512898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,768,0.018310399850209554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,512,0.007524266839027405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,512,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,128,0.005689600110054016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,256,0.004199466605981191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,256,0.015357866883277893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,8192,0.0464789350827535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,128,0.003489066660404205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,7168,0.0365610678990682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,128,0.015149866541226705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,64,0.003303466737270355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,5120,0.027960532903671266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,64,0.014924800395965577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,32,0.015024000406265258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,2048,0.014139733711878457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,65536,0.4875637372334798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,1536,0.011796266833941142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,65536,0.2602389335632324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,768,0.008528000116348267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,51200,0.38435519536336266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,32,0.0032970666885375976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,512,0.006545066833496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,16384,0.12784000237782794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,16384,0.07994773387908935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,51200,0.20932159423828126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,16384,0.06491413513819376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,51200,0.18680960337320965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,12288,0.06152213414510092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,12288,0.09789013067881266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,10240,0.08262613614400229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,256,0.005736533304055532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,10240,0.04231893221537272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,8192,0.0684063990910848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,8192,0.04580373366673787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,7168,0.05618133147557577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,7168,0.04176959991455078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,6144,0.047670400142669676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,6144,0.03861653407414754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,6144,0.027834665775299073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,5120,0.039930665493011476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,5120,0.034782934188842776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,5120,0.024175999561945598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,4096,0.032840534051259355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,4096,0.030280532439549764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,128,0.005042133231957754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,4096,0.02066453297932943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,3584,0.028269867102305096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,10240,0.05559360186258951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,3072,0.025538132588068647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,3072,0.026195200284322102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,65536,0.23981547355651855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,3584,0.029014400641123456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,2560,0.024408533175786336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,2560,0.02480640014012655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,2560,0.014716800053914389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,2048,0.022409600019454957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,12288,0.05649066766103109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,1536,0.013963733116785684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,1536,0.020567466815312706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,1024,0.010382933417956035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,8192,0.03563626607259114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,1024,0.018959999084472656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,768,0.008612266182899475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,7168,0.03146773378054301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,768,0.017211733261744182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,512,0.0068800002336502075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,512,0.015607466300328573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,2048,0.017934934298197428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,256,0.003753600021203359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,256,0.015156267086664834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,3584,0.018547199169794717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,128,0.003387733300526937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,128,0.0151146670182546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,3072,0.01646293302377065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,64,0.0031061333914597826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,64,0.015153066317240397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,32,0.0031178665657838186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,32,0.014915200074513755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,65536,0.3783733367919922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,65536,0.20820159912109376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,1536,0.010613333185513813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,65536,0.19466880162556965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,1024,0.00894719958305359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,768,0.0073173334201176955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,51200,0.2994122823079427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,51200,0.155842129389445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,51200,0.16690239906311036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,16384,0.09361279805501302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,512,0.006134399771690368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,12288,0.07156053384145102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,16384,0.07041707038879394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,12288,0.0515008012453715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,12288,0.04174506664276123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,10240,0.059845332304636636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,10240,0.04557013511657715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,8192,0.04897600015004476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,8192,0.0394976019859314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,7168,0.04314133326212565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,7168,0.036294400691986084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,6144,0.037829331556955975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,2048,0.012558933099110922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,6144,0.03328106602032979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,256,0.0054954667886098225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,5120,0.032035199801127116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,5120,0.030237867434819536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,4096,0.02651840051015218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,4096,0.02696746587753296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,4096,0.017520000537236534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,128,0.004825599988301595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,3584,0.02350506583849589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,3584,0.016405333081881204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,3584,0.0256223996480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,3072,0.02066133419672648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,3072,0.02373653252919515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,2560,0.01772586703300476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,2560,0.022451200087865195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,16384,0.05354453325271606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,2048,0.014640000462532044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,2048,0.020855466524759926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,10240,0.035813331604003906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,2048,0.011175466577212016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,1536,0.011617066462834676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,8192,0.030164267619450885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,1536,0.01954666574796041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,1024,0.008689066767692566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,1536,0.009853866696357728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,768,0.00745066652695338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,1024,0.008133333424727123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,1024,0.017796266078948974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,768,0.006852266689141591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,512,0.005924266576766968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,512,0.015482667088508605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,6144,0.02372480034828186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,256,0.003655466685692469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,5120,0.020660267273585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,256,0.015310933192571005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,128,0.003339733431736628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,128,0.014917332927385965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,128,0.005097599824269613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,64,0.0031541332602500914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,64,0.015166933337847391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,3072,0.014193066954612732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,32,0.0031786667803923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,32,0.015156267086664834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,2560,0.012970667084058127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,65536,0.1945013364156087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,65536,0.33512105941772463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,7168,0.026899200677871705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,768,0.015656532843907674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,51200,0.15730239550272623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,51200,0.2655669371287028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,16384,0.08906773726145426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,12288,0.06479253371556601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,16384,0.057393066088358556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,12288,0.04809066851933797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,12288,0.0398911992708842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,10240,0.05470293362935384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,10240,0.04222826560338338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,10240,0.034770135084788004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,8192,0.04344000021616618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,8192,0.03677440087000529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,7168,0.03935786485671997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,7168,0.033861335118611655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,512,0.006205866734186808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,6144,0.03364906708399455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,6144,0.031301333506902056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,6144,0.022680532932281495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,256,0.005614933371543884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,5120,0.028933332363764448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,5120,0.028538666168848675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,4096,0.02391786575317383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,5120,0.019715199867884316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,3584,0.021321600675582884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,4096,0.01667733391125997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,4096,0.02534613410631816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,3072,0.018795732657114664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,3584,0.015239466230074564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,3584,0.024565333127975465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,2560,0.016264533003171287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,3072,0.013662933309872945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,3072,0.02318933407465617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,2048,0.013211733102798462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,2560,0.02169493238131205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,2048,0.010618666807810467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,2048,0.020106667280197145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,1536,0.010819199681282043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,1536,0.018732800086339315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,1536,0.009225599964459737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,1024,0.008228266735871632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,1024,0.016548267006874083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,1024,0.007136000196139018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,768,0.006931200126806895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,768,0.015481600165367126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,512,0.004374399781227112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,512,0.015273599823315939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,512,0.005924266576766968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,256,0.0035264000296592714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,256,0.015005866686503092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,256,0.005234133203824361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,65536,0.1863317330678304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,128,0.00317546675602595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,128,0.014923733472824097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,51200,0.15015679995218914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,64,0.003013333429892858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,16384,0.052130134900410977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,32,0.0030293333033720653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,64,0.014834133783976236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,65536,0.2829450607299805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,8192,0.029496532678604127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,65536,0.1634218692779541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,7168,0.025897600253423053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,51200,0.1318720022837321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,51200,0.2237845261891683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,65536,0.17811412811279298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,16384,0.07737279733022054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,16384,0.054258131980896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,12288,0.06165013313293457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,51200,0.14351040522257488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,12288,0.04446719884872437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,10240,0.051254399617513026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,10240,0.039661868413289385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,10240,0.033181866010030106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,8192,0.040201600392659506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,8192,0.034828801949818924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,7168,0.03509120146433513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,32,0.014754133423169455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,7168,0.03224746584892273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,6144,0.03076480031013489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,2560,0.012330666184425354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,6144,0.02986026604970296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,6144,0.021240532398223877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,768,0.0064533332983652755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,5120,0.02578666607538859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,5120,0.02765013376871745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,4096,0.022094933191935222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,8192,0.027858134110768634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,4096,0.01564479966958364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,3584,0.019720532496770225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,3584,0.02350826660792033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,128,0.004875733455022176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,3072,0.01740586757659912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,3072,0.022290132443110146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,3072,0.013011200229326883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,2560,0.015105066696802774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,2560,0.021186133225758873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,2560,0.011896533767382304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,2048,0.012599466244379678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,2048,0.019732266664505005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,4096,0.02481600046157837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,1536,0.0101173331340154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,12288,0.03765120108922322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,1536,0.018506666024525963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,1024,0.007546666761239369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,1024,0.01591253379980723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,3584,0.014245333274205527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,768,0.006472533444563548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,768,0.01530026694138845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,7168,0.02422399918238322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,512,0.0038901334007581077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,512,0.01516480048497518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,256,0.003368533402681351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,256,0.014717866977055868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,16384,0.04941226641337077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,128,0.003151999910672506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,128,0.014924800395965577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,5120,0.01841920018196106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,64,0.002995199958483378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,64,0.014668800433476768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,32,0.003048533449570338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,32,0.014837333559989929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,65536,0.2352959950764974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,2048,0.010200533270835876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,65536,0.14483413696289063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,65536,0.16584107081095378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,51200,0.11374613444010417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,1536,0.008968533078829447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,16384,0.06538666486740112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,1024,0.006853333115577698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,16384,0.048249598344167074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,12288,0.050581332047780356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,768,0.006230400005976359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,12288,0.038736001650492353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,10240,0.04306986729303996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,10240,0.03499626715977987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,10240,0.03102933367093404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,8192,0.035214932759602864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,8192,0.030857600768407184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,256,0.00510506679614385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,8192,0.02642986575762431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,7168,0.029554132620493574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,7168,0.028460800647735596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,6144,0.025487999121348064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,6144,0.02681386669476827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,51200,0.18552319208780926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,5120,0.023512534300486245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,128,0.004648533463478088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,5120,0.024655999739964803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,4096,0.01798293391863505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,4096,0.022334933280944824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,4096,0.015011200308799743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,3584,0.016062933206558227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,3584,0.02155839999516805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,512,0.005639466643333435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,3072,0.01421119968096415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,3072,0.021363200743993123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,3072,0.012489599982897441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,2560,0.012199466427167256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,2560,0.020104533433914183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,2560,0.011315199732780456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,2048,0.010417067011197408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,2048,0.018905599912007652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,2048,0.009690666198730468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,1536,0.008697600166002909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,1536,0.017933867375055947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,51200,0.1333877404530843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,16384,0.04580053488413493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,1024,0.007551999886830647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,1024,0.015677866339683533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,768,0.005191466708978017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,768,0.015411200126012168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,768,0.006061866879463196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,512,0.003701333453257879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,512,0.01508799990018209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,12288,0.03540800015131633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,256,0.003304533412059148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,256,0.015034666657447815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,7168,0.023150932788848878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,128,0.0030517332255840302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,6144,0.02028053402900696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,128,0.014459733168284097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,128,0.004691199958324432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,64,0.0029535998900731405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,5120,0.017395200332005818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,64,0.014521599809328715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,32,0.0030016000072161358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,32,0.014586666226387024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,65536,0.19035199483235676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,65536,0.11644266446431477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,3584,0.013704533378283182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,51200,0.09439679781595865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,65536,0.14301013946533203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,51200,0.15060480435689289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,16384,0.05130666494369507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,16384,0.04133439858754476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,12288,0.04030400117238363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,12288,0.03455359935760498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,1536,0.008296533425649007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,10240,0.03408426841100057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,10240,0.031146667400995892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,1024,0.006671999891599019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,8192,0.02842560013135274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,8192,0.027748266855875652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,8192,0.02418986757596334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,7168,0.025552000602086383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,512,0.0055402666330337524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,7168,0.02572159965833028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,7168,0.02058666745821635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,6144,0.02454293370246887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,256,0.00510506679614385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,5120,0.01827733318010966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,5120,0.02259733279546102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,5120,0.01609386702378591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,4096,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,4096,0.02127466599146525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,3584,0.013476266463597616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,3584,0.020444800456364952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,6144,0.02286400000254313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,3072,0.011924266815185547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,3072,0.020207999149958293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,51200,0.11494932969411213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,2560,0.01048959990342458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,16384,0.041653335094451904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,2560,0.019423999389012656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,2048,0.008940800031026205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,12288,0.032738133271535234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,2048,0.018106667200724284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,2048,0.009237333138783773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,1536,0.007575466732184092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,1536,0.00764160007238388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,1536,0.01567466656366984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,1024,0.006004266440868378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,1024,0.015225600202878317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,1024,0.006539733211199443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,768,0.0039594667653242745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,768,0.015089066823323569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,512,0.0035829332967599234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,512,0.014801067113876343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,6144,0.018472532431284584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,256,0.0031797334551811217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,256,0.014818132917086283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,10240,0.028168533245722455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,128,0.002942933390537898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,128,0.014694399634997048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,4096,0.013500799735387167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,64,0.002842666705449422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,64,0.014679466684659323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,32,0.0029535998900731405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,32,0.014940800269444785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,3584,0.012852266430854797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,65536,0.17317333221435546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,65536,0.10798400243123371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,3072,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,51200,0.1387114683787028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,2560,0.01048533320426941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,51200,0.08798293272654215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,16384,0.03845333258310954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,16384,0.04565653403600057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,12288,0.036158935228983564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,12288,0.03289706707000732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,768,0.00606826643149058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,10240,0.030858665704727173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,10240,0.029969066381454468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,512,0.005534933507442474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,8192,0.025646932919820148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,8192,0.026009599367777508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,8192,0.022437334060668945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,7168,0.023167999585469563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,7168,0.024562132358551026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,51200,0.10666986306508382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,6144,0.02103040019671122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,256,0.0050805335243542995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,6144,0.02332373261451721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,5120,0.01803413430849711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,5120,0.021826134125391642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,5120,0.015335466464360556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,4096,0.013697066903114319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,128,0.004655999938646952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,4096,0.020522665977478028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,65536,0.13207999865214032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,3584,0.01363200048605601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,3584,0.019725867112477622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,3584,0.012469333410263062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,3072,0.010804266730944315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,3072,0.019350399573644005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,3072,0.011321600278218586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,2560,0.009622400005658466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,16384,0.03893440167109172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,2560,0.018556799491246542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,2048,0.008422399560610454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,2048,0.016706132888793947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,12288,0.029915734132130938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,2048,0.00832533339659373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,1536,0.007245866457621257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,1536,0.016105600198109946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,1024,0.004922666649023692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,1024,0.015559466679890952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,10240,0.02635519901911418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,1024,0.006306133170922597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,768,0.003967999915281932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,768,0.015186132987340293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,512,0.003553066651026408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,512,0.015020799636840821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,512,0.005352533360322317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,256,0.00315733328461647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,256,0.014672000209490457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,256,0.005020800232887268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,7168,0.019610667228698732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,128,0.0029813334345817565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,128,0.014570666352907815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,64,0.002926933268706004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,64,0.014355199535687766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,32,0.0029919999341169994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,32,0.014424533645311991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,6144,0.017678932348887125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,65536,0.14667840003967286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,65536,0.09506986935933431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,4096,0.013338667154312134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,51200,0.11649813652038574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,51200,0.0772320032119751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,16384,0.0415882666905721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,16384,0.036697598298390706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,12288,0.03269973397254944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,12288,0.030618667602539062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,12288,0.03184319933255513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,10240,0.027808000644048054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,10240,0.027960532903671266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,10240,0.027376000086466474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,8192,0.023825067281723022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,8192,0.025565866629282636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,8192,0.023238400618235268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,7168,0.02144213318824768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,7168,0.02325973312060038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,2560,0.010259200135866802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,7168,0.020409599939982096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,6144,0.019141334295272826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,6144,0.02240426739056905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,5120,0.0165503998597463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,1536,0.0074986666440963745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,5120,0.02103360096613566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,4096,0.014265599846839904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,768,0.00574186642964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,4096,0.019799466927846274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,3584,0.01120960017045339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,4096,0.013503999511400858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,3584,0.01951040029525757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,3072,0.010186666250228881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,3072,0.018807466824849448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,3072,0.011485866705576579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,2560,0.008975999553998311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,2560,0.017416532834370932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,128,0.004745600124200186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,2048,0.007841066519419352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,2048,0.016178133090337117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,65536,0.15227200190226237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,1536,0.006795733173688252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,1536,0.015986133615175882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,1536,0.007464533547560374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,1024,0.004316799839337667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,1024,0.015378133455912272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,1024,0.00633493314186732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,768,0.003944533318281174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,51200,0.12118720213572184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,16384,0.04075839916865031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,768,0.015228799978892007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,512,0.00359253336985906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,768,0.00584853341182073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,512,0.0149536003669103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,256,0.0031925333042939507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,256,0.014857600132624308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,128,0.0030250666042168934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,128,0.014614400267601014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,128,0.004643199841181437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,64,0.0028160000840822858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,64,0.014670933286348978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,32,0.0028768000503381092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,32,0.014755200346310934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,65536,0.12577280203501384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,65536,0.0832266648610433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,6144,0.018693333864212035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,5120,0.015635200341542563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,3584,0.012645333011945089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,51200,0.10056533018747967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,51200,0.0662773331006368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,16384,0.034409598509470625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,16384,0.0322922666867574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,2560,0.010174933075904845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,12288,0.02729066610336304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,12288,0.02777600089708964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,512,0.0052490666508674625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,10240,0.023749333620071412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,256,0.005036800106366476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,10240,0.025864533583323163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,8192,0.019735467433929444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,8192,0.02371306618054708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,2048,0.008342400193214417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,7168,0.018017067511876424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,7168,0.02282879948616028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,65536,0.13574934005737305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,6144,0.015821866194407144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,6144,0.021647999684015908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,5120,0.013885866602261862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,6144,0.0166485329469045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,5120,0.020498132705688475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,5120,0.014586666226387024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,4096,0.011959466338157653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,51200,0.10690879821777344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,4096,0.012589866916338602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,4096,0.019745065768559774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,3584,0.011158399780591329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,3584,0.011573333541552227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,3072,0.01032960017522176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,3072,0.01051093339920044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,3072,0.01863573392232259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,2560,0.008328533172607422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,2560,0.009096533060073853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,2048,0.007309866448243459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,2048,0.01639573375384013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,12288,0.02841920057932536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,1536,0.0059008002281188965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,1536,0.015703466534614564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,3584,0.019103999932607016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,1024,0.004289066791534424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,1024,0.015405866503715514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,10240,0.02416320045789083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,2560,0.017234132687250773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,768,0.0038602667550245917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,768,0.015314132968584696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,512,0.003555200000603994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,512,0.014993066589037577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,8192,0.021512534221013388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,512,0.005239466826121012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,256,0.0032330666979153953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,256,0.014798933267593383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,128,0.002977066735426585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,128,0.014354133605957031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,7168,0.018620800971984864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,64,0.0027669332921504974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,64,0.014567466576894126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,32,0.0029077333708604175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,32,0.014659200112024942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,65536,0.10029653708140056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,65536,0.06854399840037027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,16384,0.037006934483846024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,51200,0.0773973306020101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,51200,0.05728000005086263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,2048,0.007926400005817413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,16384,0.02823466658592224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,16384,0.028701867659886676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,1536,0.007049599786599477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,12288,0.022769065697987874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,12288,0.025550933678944905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,768,0.005645866692066193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,1024,0.006131199995676676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,10240,0.01984213391939799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,10240,0.023553067445755006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,10240,0.024551467100779215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,8192,0.016173866391181946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,8192,0.022087466716766358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,256,0.005042133231957754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,7168,0.014757333199183145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,7168,0.01881600022315979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,7168,0.021258666117986044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,6144,0.013329066832860312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,6144,0.020694400866826376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,5120,0.011954133709271748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,128,0.004698666433493296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,5120,0.02021013299624125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,4096,0.010420266787211101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,4096,0.019019732872645058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,4096,0.012380799651145935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,3584,0.00976213316122691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,3584,0.01775040030479431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,65536,0.1263818661371867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,3072,0.009221333265304565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,3072,0.016593066851298015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,51200,0.10250346660614014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,2560,0.0076330666740735366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,2560,0.01629973351955414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,16384,0.036514135201772054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,12288,0.028402133782704668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,2048,0.006746666630109151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,2048,0.01649173299471537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,2048,0.008094933132330577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,1536,0.004877866804599762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,1536,0.015782399972279867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,8192,0.021781333287556968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,1024,0.004196266829967499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,1024,0.01539413332939148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,1024,0.006234666705131531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,768,0.0037920000652472176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,6144,0.016857600212097167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,768,0.01527679959932963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,512,0.003403733422358831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,512,0.01492586632569631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,5120,0.014637866616249084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,256,0.0031530665854612983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,256,0.014613333344459533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,3584,0.011413333813349406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,128,0.002940800040960312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,128,0.014636799693107605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,128,0.0046847999095916745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,64,0.0027445333699385325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,64,0.014617600043614707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,32,0.0027317332724730173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,32,0.014514133334159851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,3072,0.010419199864069622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,65536,0.07744106451670328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,65536,0.05874346494674683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,51200,0.06181973218917847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,51200,0.04860266844431559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,2560,0.009325866897900898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,16384,0.02297813296318054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,16384,0.02580159902572632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,12288,0.018497065703074137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,12288,0.022973867257436116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,1536,0.00714026689529419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,10240,0.016140799721082053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,10240,0.022205867369969687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,768,0.005621333420276642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,8192,0.013427199920018515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,8192,0.02071146567662557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,512,0.005458133419354757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,8192,0.019288533926010133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,7168,0.012594133615493774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,7168,0.01606613298257192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,6144,0.011520000298817952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,6144,0.01938986579577128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,256,0.0050570666790008545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,5120,0.010212266445159912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,5120,0.0191103994846344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,5120,0.012995200355847678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,4096,0.009076266487439474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,4096,0.01730026602745056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,65536,0.10886613527933757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,3584,0.00879146655400594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,3584,0.01726079980532328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,51200,0.0870304028193156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,3072,0.008400000135103862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,7168,0.02042133410771688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,3072,0.016901334126790367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,3072,0.008794666330019633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,2560,0.0071263998746871945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,2560,0.01632533371448517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,2048,0.005526400109132131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,16384,0.030393600463867188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,2048,0.016059733430544534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,2048,0.007152000069618225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,1536,0.015802666544914246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,1536,0.0067093332608540845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,1024,0.004152533411979675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,12288,0.02409279942512512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,10240,0.02077546715736389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,768,0.0037685332198937735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,768,0.015291733543078103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,1536,0.0048437332113583885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,768,0.005404800176620483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,512,0.003436800092458725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,512,0.014905599753061929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,256,0.003124266614516576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,6144,0.014569600423177084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,256,0.014716800053914389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,256,0.004839466512203216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,1024,0.015457066893577575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,128,0.0028959999481836954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,64,0.002855466554562251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,64,0.014570666352907815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,32,0.0028885332246621448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,32,0.014465066790580749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,4096,0.010227200388908387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,65536,0.051918931802113856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,65536,0.045534932613372804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,51200,0.04171093304951985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,51200,0.0387168010075887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,3584,0.00969493289788564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,16384,0.0164000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,16384,0.02260800004005432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,16384,0.0253983994325002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,12288,0.013659733533859252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,2560,0.007919999957084655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,12288,0.020885332425435384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,128,0.014622933665911355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,12288,0.019875200589497884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,10240,0.012221866846084594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,10240,0.020461867252985634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,8192,0.010569600264231364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,8192,0.019358932971954346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,1024,0.00588266650835673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,8192,0.015029333035151162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,7168,0.009915733337402343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,7168,0.018548266092936198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,6144,0.008985599875450135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,6144,0.017964800198872886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,512,0.005047466854254405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,5120,0.008654933174451191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,5120,0.017812265952428182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,5120,0.010201600193977357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,128,0.004598399996757508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,4096,0.007773866752783458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,4096,0.008660266796747845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,4096,0.017493333419164023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,3584,0.00739519993464152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,3584,0.0173962672551473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,3072,0.006635733445485433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,3072,0.016953599452972413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,65536,0.08752000331878662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,2560,0.006065066655476888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,2560,0.01669013301531474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,51200,0.07091413338979086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,2048,0.005358933409055074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,2048,0.01604480048020681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,1536,0.004749866823355356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,1536,0.015633066495259605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,10240,0.017292799552281697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,1024,0.004089600096146265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,1024,0.015160533785820007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,7168,0.01360319952170054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,768,0.0038101332883040107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,768,0.015056000153223673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,6144,0.011306666334470113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,512,0.0033845332761605583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,512,0.014945066968599954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,512,0.004981333514054617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,256,0.003058133274316788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,256,0.01476800044377645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,3584,0.008213333288828532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,128,0.0028927999238173166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,128,0.014411733547846476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,3072,0.007686399916807811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,64,0.00276053324341774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,64,0.014537599682807923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,32,0.0027722666660944624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,32,0.014517333110173544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,65536,0.04105493227640788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,2560,0.00728000005086263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,65536,0.04059520165125529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,51200,0.033557331562042235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,2048,0.006545066833496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,51200,0.03501760164896647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,16384,0.013699199755986533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,16384,0.021553067366282146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,1536,0.00609493354956309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,12288,0.011664000153541566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,12288,0.020142932732899986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,1024,0.0053610667586326596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,10240,0.0106495996316274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,10240,0.019130667050679527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,10240,0.015740799903869628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,8192,0.009467732906341553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,8192,0.018205867211023966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,768,0.005160533388455709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,8192,0.013085866967837015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,7168,0.008775466680526733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,7168,0.01848319967587789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,7168,0.011979732910792034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,6144,0.007957333326339721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,6144,0.01070186694463094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,6144,0.017910399039586387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,5120,0.007729066908359528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,5120,0.017861332496007284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,4096,0.006585599978764851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,256,0.004694400231043497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,4096,0.008505599697430928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,4096,0.017799466848373413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,3584,0.0070250665148099255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,3584,0.017358932892481485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,3072,0.00625600020090739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,3072,0.01691626707712809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,3072,0.007507200042406718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,2560,0.006123733520507812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,2560,0.01660053332646688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,128,0.0045503998796145115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,2048,0.005412266651789347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,2048,0.01599253316720327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,65536,0.07612693309783936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,1536,0.004760533571243286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,1536,0.015650133291880287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,51200,0.06161173184712728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,1024,0.004098133246103922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,16384,0.022507733106613158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,1024,0.015332266688346863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,12288,0.017911465962727864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,768,0.0037237333754698435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,768,0.014919466773668923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,5120,0.00957546631495158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,512,0.00342399999499321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,512,0.014758400122324624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,3584,0.008063999811808269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,256,0.003052799900372823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,256,0.014642133315404256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,2560,0.007131733496983846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,128,0.0028480000793933867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,128,0.01455466647942861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,2048,0.006320000191529592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,64,0.0027413333455721537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,64,0.01437440017859141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,32,0.0027456000447273255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,32,0.014405333002408347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,65536,0.02951893409093221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,65536,0.034125868479410806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,1536,0.005919999877611796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,51200,0.024495999018351235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,51200,0.028889600435892743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,1024,0.005330133438110352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,16384,0.010726400216420491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,16384,0.0199455996354421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,12288,0.0092031995455424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,768,0.00506986677646637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,12288,0.01798080007235209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,10240,0.008770133058230083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,10240,0.01808639963467916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,10240,0.014850133657455444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,8192,0.008065066734949748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,8192,0.01803626616795858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,512,0.004931200047334035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,7168,0.006911999980608622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,7168,0.018387200435002644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,256,0.004676266511281332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,6144,0.006486399968465169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,6144,0.01834453344345093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,128,0.004442666471004486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,5120,0.006514133512973785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,5120,0.018173867464065553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,65536,0.07552320162455241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,4096,0.006110933423042297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,4096,0.017781333128611247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,51200,0.05997226635615031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,3584,0.006479999919732411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,3584,0.01720213294029236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,16384,0.022343466679255165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,3072,0.006226133306821187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,3072,0.0168778657913208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,12288,0.01713493267695109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,2560,0.006049066781997681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,2560,0.01639359990755717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,8192,0.012699733177820841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,2048,0.0053162669142087305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,2048,0.015803733468055726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,7168,0.011594667037328085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,1536,0.004762666424115499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,1536,0.015686399737993875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,6144,0.010558933019638062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,1024,0.004070399949947992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,1024,0.015399466951688132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,5120,0.009642666578292847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,768,0.003703466554482778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,768,0.015086932977040609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,4096,0.008427733182907104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,512,0.003387733300526937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,512,0.014959999918937683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,3584,0.008016000191370647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,256,0.003054933249950409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,256,0.014412800470987955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,3072,0.0074421331286430355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,128,0.0028266665836175283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,128,0.014432000120480857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,2560,0.0070709332823753355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,64,0.002666666607062022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,64,0.014381866653760275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,32,0.0026858667532602947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,32,0.014387200276056925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,65536,0.018526933590571084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,65536,0.026632533470789595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,2048,0.006312533219655354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,51200,0.015406933426856995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,51200,0.023653332392374674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,1536,0.005734399954477946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,16384,0.007794133325417836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,16384,0.018994132677714028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,16384,0.02141759991645813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,12288,0.006607999900976817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,12288,0.018191999197006224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,1024,0.005284266670544943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,10240,0.006679466863473256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,10240,0.018492799997329713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,768,0.004984533290068308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,8192,0.006617600222428639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,8192,0.017892267306645712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,8192,0.012648533781369528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,512,0.0047989333669344585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,7168,0.006230400005976359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,7168,0.01840106646219889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,7168,0.011769599715868632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,6144,0.006027733286221823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,6144,0.01783039967219035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,6144,0.010563199718793232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,256,0.004682666560014089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,5120,0.006450133522351582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,5120,0.00953493316968282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,5120,0.017861332496007284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,4096,0.006030933558940887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,128,0.00453653335571289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,3584,0.006479999919732411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,3584,0.017116800944010416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,3584,0.007939200103282928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,3072,0.006144000093142191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,3072,0.016694400707880655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,3072,0.007388799885908763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,65536,0.07514986991882325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,2560,0.006060799956321717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,2560,0.01651306649049123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,2048,0.005332266787687937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,51200,0.05939626693725586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,2048,0.015994667013486227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,1536,0.004669866462548574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,1536,0.015495466192563376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,1536,0.005926399926344554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,1024,0.0040224000811576845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,1024,0.015258666872978211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,12288,0.016862932840983072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,768,0.0037962667644023894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,4096,0.01737920045852661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,768,0.015156267086664834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,512,0.003436800092458725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,512,0.014898133277893067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,10240,0.014850133657455444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,256,0.003020799905061722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,256,0.014703999956448874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,128,0.002869333326816559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,128,0.0144896000623703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,128,0.004404266675313314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,64,0.002643200010061264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,64,0.014404267072677612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,32,0.002639999985694885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,32,0.014430933197339377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,65536,0.011993599931399028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,65536,0.024309333165486655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,65536,0.07452053229014079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,51200,0.010732799768447876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,51200,0.021406932671864828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,4096,0.008506666620572407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,16384,0.006212266782919565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,16384,0.01853546698888143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,16384,0.0214464008808136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,12288,0.006029866635799408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,12288,0.018232532342274985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,2560,0.006983466446399689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,10240,0.0061749334136645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,10240,0.018823466698328652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,2048,0.006287999947865804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,8192,0.006179200112819671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,10240,0.014818132917086283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,8192,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,8192,0.017914666732152303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,7168,0.005989333490530649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,7168,0.01827733318010966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,6144,0.005952000121275584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,1024,0.005161599814891815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,6144,0.017832533518473307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,5120,0.00626986672480901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,5120,0.01802133321762085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,768,0.004935466746489207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,4096,0.006002133091290792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,512,0.004811733464399974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,4096,0.01726613243420919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,3584,0.006365866462389629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,3584,0.016977065801620485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,256,0.004522666831811269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,3072,0.005970133344332377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,3072,0.01676586667696635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,51200,0.05875626802444458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,2560,0.005961599946022034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,2560,0.016337066888809204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,2560,0.006852266689141591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,2048,0.005332266787687937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,2048,0.01583146651585897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,12288,0.016874667008717856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,1536,0.004706133405367533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,1536,0.015622400244077048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,7168,0.011716266473134358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,1024,0.004003199934959412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,6144,0.010679466525713603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,1024,0.015371732910474143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,1024,0.005274666845798493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,768,0.003667200108369192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,768,0.014999467134475707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,768,0.004974933465321859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,512,0.0033301333586374915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,512,0.014907733599344889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,5120,0.009481599926948548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,256,0.0030293333033720653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,4096,0.008380800485610962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,256,0.014533332983652749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,256,0.004604800045490265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,128,0.002855466554562251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,128,0.00456639975309372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,128,0.01441493332386017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,64,0.0027935999135176343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,32,0.0027061333258946735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,3584,0.008137600123882293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,32,0.01444906691710154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,65536,0.00890880028406779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,51200,0.008884267012278239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,65536,0.023435733715693154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,16384,0.006196266909440359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,51200,0.02100800077120463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,12288,0.0060138667623202005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,16384,0.01831679940223694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,3072,0.007379200061162312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,10240,0.006241066753864289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,12288,0.017883733908335368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,10240,0.018437333901723228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,8192,0.006053333481152853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,7168,0.005901866654555003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,8192,0.017705599466959633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,6144,0.005763199925422668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,7168,0.018307199080785118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,5120,0.0061141331990559895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,6144,0.01763413349787394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,4096,0.005672533313433329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,5120,0.017997866868972777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,3584,0.006197333335876465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,4096,0.017387733856836955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,3072,0.005845333139101664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,3584,0.017038933436075845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,2560,0.006004266440868378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,3072,0.01676586667696635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,2048,0.0054848000407218935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,2560,0.016539733608563742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,1536,0.00462719996770223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,2048,0.015745066603024802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,1024,0.003978666663169861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,1536,0.015559466679890952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,768,0.003692800054947535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,1024,0.015049599607785544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,512,0.0033471999069054925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,768,0.014994133512179056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,256,0.002922666569550832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,512,0.014731733004252115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,128,0.002776533365249634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,256,0.014418133099873862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,64,0.0026506667335828146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,128,0.01446399986743927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,32,0.0027221334477265675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,64,0.01416106621424357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,65536,0.008714666962623597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,32,0.014486400286356607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,51200,0.008476799726486206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,65536,0.022898133595784506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,16384,0.00645653357108434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,51200,0.020679465929667153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,12288,0.006331733365853627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,16384,0.018228266636530557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,10240,0.006192000210285186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,12288,0.01799573302268982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,8192,0.0059349333246548975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,10240,0.01834133267402649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,7168,0.005843199789524078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,8192,0.017975467443466186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,6144,0.005735466877619425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,7168,0.01791999936103821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,2048,0.006265600025653839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,5120,0.006064000229040781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,6144,0.01767786741256714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,4096,0.005704533557097117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,3584,0.006232533355553945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,5120,0.01785599986712138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,4096,0.017770665884017944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,3072,0.00591893345117569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,2560,0.006005333364009857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,3584,0.016978132724761962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,2048,0.005524266759554545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,2560,0.016383999586105348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,1536,0.004678399860858917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,2048,0.015661866466204325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,1536,0.005736533304055532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,1024,0.003934933245182038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,1536,0.015402666727701821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,768,0.003622400015592575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,512,0.0033813332517941795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,1024,0.015066666404406228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,768,0.014988799889882406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,256,0.0029696000119050344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,512,0.014816000064214071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,128,0.0028394666810830434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,256,0.014525866508483887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,64,0.002648533384005229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,128,0.014468266566594442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,32,0.0027082666754722597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,64,0.014413866400718688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,32,0.014437333742777506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,512,0.004714666803677877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,64,0.01458346645037333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,3072,0.01660480002562205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,16384,0.39358720779418943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,16384,0.7505184173583984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,12288,0.299179744720459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,12288,0.5671146392822266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,51200,1.1822186787923177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,10240,0.2513024012247721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,10240,0.48103892008463545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,51200,2.361268361409505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,8192,0.37879467010498047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,8192,0.20849173863728843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,7168,0.1810090700785319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,7168,0.3308650652567545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,6144,0.28612372080485027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,6144,0.15811626116434735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,5120,0.23816426595052084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,5120,0.13445013364156086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,12288,0.29981759389241536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,16384,0.3592725435892741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,4096,0.2057706673940023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,4096,0.11040960152943928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,51200,1.098525873819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,3584,0.16852480570475262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,3584,0.11397226651509602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,10240,0.26237546602884926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,3072,0.14686400095621746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,3072,0.09542293548583984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,2560,0.12224213282267253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,2560,0.07610666751861572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,7168,0.1608693281809489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,6144,0.1392298698425293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,2048,0.09807466665903727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,8192,0.2125216007232666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,2048,0.06626559893290201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,1536,0.08778026898701986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,1536,0.05232959985733032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,1024,0.05160319805145264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,1024,0.04040106534957886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,5120,0.11710293292999267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,768,0.04387413263320923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,768,0.034568532307942705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,4096,0.09544853369394937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,512,0.028099199136098225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,512,0.02776319980621338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,3072,0.07348159948984781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,256,0.01794559955596924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,3584,0.08421119848887125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,256,0.02146453261375427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,128,0.009329066673914591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,128,0.018550399939219156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,1536,0.04031039873758952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,2560,0.0626965324083964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,64,0.006567466755708058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,65536,32,0.004837333162625631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,64,0.0159850666920344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,65536,32,0.0168287992477417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,2048,0.051319468021392825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,1024,0.029334400097529096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,768,0.023481599489847817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,65536,1.2207530975341796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,51200,1.8706612904866535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,65536,2.3790206909179688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,16384,0.7110997517903646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,128,0.009638399879137675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,256,0.012476799885431926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,16384,0.37147092819213867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,65536,512,0.017644800742467246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,12288,0.23983999888102212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,12288,0.4743317286173503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,51200,0.960761578877767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,10240,0.38652159372965494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,10240,0.2012341340382894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,8192,0.1659338633219401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,8192,0.3527807871500651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,7168,0.2577343940734863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,7168,0.1470666726430257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,6144,0.22685866355895995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,6144,0.1287274678548177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,65536,1.1719712575276693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,5120,0.2182688077290853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,5120,0.12374186515808105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,12288,0.22660479545593262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,51200,0.9171328226725259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,4096,0.1531925360361735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,16384,0.3188234647115072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,4096,0.10165546735127766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,3584,0.15398826599121093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,3584,0.09096533457438151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,10240,0.19078186353047688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,3072,0.1153760035832723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,3072,0.07889599800109863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,8192,0.15347733497619628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,2560,0.06248426834742228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,6144,0.11753386656443279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,7168,0.1354090690612793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,2048,0.05339093208312988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,2048,0.08463679949442546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,2560,0.09523306687672933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,1536,0.05877866744995117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,1536,0.044683734575907394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,1024,0.04049493471781413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,1024,0.034620801607767746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,5120,0.09908693631490072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,768,0.03561919927597046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,768,0.02948266665140788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,4096,0.08115413188934326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,3072,0.062223998705546055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,512,0.02498133381207784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,512,0.02412160038948059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,256,0.012684800227483115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,256,0.01955839991569519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,3584,0.07144426504770915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,128,0.007956266899903615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,128,0.015982932845751443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,2560,0.05345493157704671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,64,0.004590933521588644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,64,0.016193067034085594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,51200,32,0.004388266801834106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,51200,32,0.0164000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,1536,0.03441173235575358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,65536,0.7440437316894531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,65536,0.3781792004903158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,2048,0.043852798144022626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,51200,0.5850666681925456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,51200,0.29982614517211914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,16384,0.18389013608296711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,16384,0.11056426366170247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,1024,0.02539520064989726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,12288,0.13951999346415203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,12288,0.08515306313832602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,768,0.020092799266179403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,256,0.011745066444079081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,10240,0.11631466547648113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,512,0.01557866632938385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,10240,0.07344106833140054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,8192,0.09460906982421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,51200,128,0.00832426647345225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,8192,0.062023464838663736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,7168,0.05644906759262085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,7168,0.08290666739145915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,6144,0.07233813603719076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,6144,0.050537598133087155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,65536,0.37837972640991213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,5120,0.07070720195770264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,5120,0.044759468237559004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,51200,0.28353811899820963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,12288,0.07274453639984131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,4096,0.057646934191385904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,16384,0.09557226498921713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,4096,0.038805333773295085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,3584,0.04783039887746175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,3584,0.03592640161514282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,3072,0.0380021333694458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,3072,0.03293866713841756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,10240,0.06170453230539957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,2560,0.03228480021158854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,2560,0.03017066717147827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,6144,0.03934079806009928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,2048,0.026434133450190227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,2048,0.0269375999768575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,7168,0.045081599553426104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,1536,0.02087786595026652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,8192,0.05049813191095988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,1536,0.024173865715662636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,1024,0.014694399634997048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,1024,0.020436267058054604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,3072,0.022760534286499025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,768,0.011593600114186604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,768,0.01918399930000305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,5120,0.03394453525543213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,512,0.008690133690834045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,512,0.017836799224217735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,3584,0.025494400660196943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,256,0.005817600091298421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,4096,0.03222080071767171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,256,0.015145599842071533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,128,0.0035786665976047516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,128,0.01504533290863037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,1536,0.01362986663977305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,64,0.0033269333342711128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,2560,0.02008533279101054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,16384,32,0.0034389334420363107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,64,0.015077333648999533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,16384,32,0.015197867155075073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,2048,0.016833066940307617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,1024,0.010824533303578694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,65536,0.2957546552022298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,65536,0.5499231974283855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,51200,0.43776534398396805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,51200,0.23639465967814127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,16384,0.1615125338236491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,16384,0.09252693653106689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,12288,0.11173653602600098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,12288,0.07092693646748861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,512,0.008010666569073994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,768,0.009292800227801006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,128,0.005031466484069824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,10240,0.09397013187408447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,16384,256,0.005550933380921682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,10240,0.06128746668497721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,8192,0.05277760028839111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,8192,0.07655039628346762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,7168,0.06686399777730306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,7168,0.0481663982073466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,65536,0.27259305318196614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,6144,0.06387840112050375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,6144,0.04302933216094971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,12288,0.056611200173695884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,5120,0.045798401037851974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,5120,0.03864959875742595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,16384,0.07370133399963379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,4096,0.03794026772181193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,4096,0.03403733174006145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,3584,0.033583998680114746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,3584,0.03159466584523519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,51200,0.2341536045074463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,3072,0.034061865011850996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,3072,0.028321067492167156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,10240,0.048232531547546385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,2560,0.02510720094045003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,2560,0.02592746615409851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,8192,0.03976320028305054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,7168,0.035485867659250894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,2048,0.021886932849884033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,2048,0.023829332987467446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,1536,0.016249600052833556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,1536,0.02181546688079834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,6144,0.031497599681218465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,1024,0.011823999881744384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,1024,0.019383466243743895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,4096,0.02304746707280477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,768,0.009987200299898785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,768,0.018397865692774455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,5120,0.027133866151173906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,512,0.00758186678091685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,512,0.015446399648984274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,3584,0.020594133933385213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,256,0.004113066693147024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,256,0.015001599987347921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,3072,0.018655999501546224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,128,0.003621333340803782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,128,0.01492586632569631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,1536,0.011335466305414836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,2560,0.016416000326474507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,64,0.0032970666885375976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,12288,32,0.0032405334214369455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,64,0.014906666676203408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,12288,32,0.015032533804575601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,2048,0.013977600137392679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,65536,0.2532255967458089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,65536,0.4608757336934407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,51200,0.20564053853352865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,51200,0.36473172505696616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,16384,0.1392181396484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,16384,0.0791701316833496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,1024,0.009589333335558574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,12288,0.09714240233103434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,12288,0.05973226626714071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,512,0.005880533158779145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,256,0.005414400001366933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,768,0.008308266599973042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,12288,128,0.004911999901135763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,10240,0.053325867652893065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,8192,0.045651201407114664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,8192,0.06704426606496175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,7168,0.05322240193684896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,7168,0.04096746842066447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,10240,0.08024960358937581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,6144,0.04648213386535645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,6144,0.03749119838078817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,65536,0.22967999776204429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,5120,0.044725334644317626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,5120,0.03373013337453206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,51200,0.18150933583577472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,16384,0.06293226480484009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,4096,0.03642666737238566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,4096,0.029816534121831256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,12288,0.04890880187352498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,3584,0.02842560013135274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,3584,0.028387200832366944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,3072,0.025090134143829344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,3072,0.025972266991933186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,10240,0.04165013233820598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,2560,0.021411200364430748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,2560,0.02406186660130819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,8192,0.03433173497517904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,2048,0.01800000071525574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,7168,0.03100693424542745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,2048,0.022333866357803343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,1536,0.014141866564750671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,1536,0.020472532510757445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,6144,0.027155200640360515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,1024,0.010190932949384054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,1024,0.018381865819295247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,5120,0.023741867144902548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,768,0.008433066805203756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,3072,0.016152532895406087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,768,0.016992000738779704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,4096,0.020460800329844157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,512,0.006781866649786632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,512,0.01558080017566681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,3584,0.01811199982961019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,256,0.0037237333754698435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,256,0.01503679951032003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,1536,0.010410666465759277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,128,0.0033600000043710076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,2560,0.014478933811187745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,128,0.01469013293584188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,64,0.003142400085926056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,64,0.014797866344451904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,10240,32,0.0031210665901501974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,10240,32,0.014823466539382935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,2048,0.01218666632970174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,1024,0.008922666311264038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,65536,0.381384531656901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,51200,0.30127251942952477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,51200,0.16312747001647948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,65536,0.202293332417806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,16384,0.09295360247294107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,16384,0.06145813465118408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,12288,0.07092053095499674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,12288,0.05073813199996948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,768,0.007012266914049785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,256,0.005320533116658529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,10240,0.059468801816304526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,512,0.005846400062243144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,10240,0.04462399880091349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,8192,0.038680533568064376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,10240,128,0.004877866804599762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,7168,0.04264320135116577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,7168,0.03592853148778279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,8192,0.04839893182118733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,6144,0.03760533332824707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,6144,0.03300906618436177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,65536,0.2588501294453939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,5120,0.03186026612917582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,5120,0.029946666955947877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,51200,0.19026560783386232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,4096,0.02986239989598592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,4096,0.026707200209299724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,16384,0.0624288002649943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,3584,0.023201066255569457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,3584,0.02518080075581868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,12288,0.04488426844278971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,3072,0.020525866746902467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,3072,0.023694932460784912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,10240,0.03775573174158732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,2560,0.017553067207336424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,8192,0.03099413315455119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,2560,0.022207999229431154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,2048,0.014471466342608133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,2048,0.020958934227625528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,6144,0.02477546731630961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,1536,0.01167680025100708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,7168,0.02807040015856425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,1536,0.019460266828536986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,1024,0.008697600166002909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,1024,0.01805013418197632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,5120,0.021684267123540244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,768,0.007268266876538594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,768,0.01579093337059021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,4096,0.018968532482782997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,512,0.005907199780146281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,512,0.015190399686495461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,3072,0.01523413360118866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,256,0.0036127999424934386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,256,0.014983466267585755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,3584,0.017077332735061644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,128,0.003278933217128118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,128,0.014711466431617738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,2560,0.01372053325176239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,64,0.00308693324526151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,64,0.014856533209482829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,8192,32,0.003159466634194056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,8192,32,0.01490133305390676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,65536,0.328443717956543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,1024,0.008483200271924337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,65536,0.18740053176879884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,768,0.006741333504517872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,51200,0.25940799713134766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,2048,0.011658666531244914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,51200,0.15156480471293132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,1536,0.010570666193962098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,16384,0.056618666648864745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,16384,0.08644159634908041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,12288,0.062882133324941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,12288,0.04690773487091064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,10240,0.0528714656829834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,10240,0.04128640095392863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,512,0.005913599828879039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,8192,0.043635201454162595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,256,0.005382399757703145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,8192,0.03640426794687907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,7168,0.038755198319753006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,7168,0.033556266625722246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,8192,128,0.0050240000089009605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,65536,0.21498452822367348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,6144,0.03380800088246663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,6144,0.031142399708429976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,51200,0.15755732854207355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,12288,0.03951679865519206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,5120,0.032553599278132124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,16384,0.05156906843185425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,4096,0.023452800512313843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,4096,0.025629866123199462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,10240,0.03376533190409343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,3584,0.021126399437586464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,3584,0.024577067295710246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,3072,0.01862186590830485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,3072,0.02344320019086202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,8192,0.02835306723912557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,2560,0.01595093309879303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,2560,0.02176426649093628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,7168,0.02637973427772522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,5120,0.028354134162267047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,2048,0.013075199723243714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,2048,0.02021119991938273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,1536,0.010846933722496033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,1536,0.018871466318766274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,6144,0.023504000902175904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,1024,0.00817920019229253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,1024,0.015564800302187601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,5120,0.020010666052500407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,768,0.006833066542943318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,768,0.015505066514015198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,4096,0.017222400506337485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,512,0.004241066674391428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,3072,0.014013866583506266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,512,0.015347199638684592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,256,0.0035018667578697203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,256,0.015098667144775391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,3584,0.015982932845751443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,128,0.003205333401759466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,2048,0.011308800180753071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,128,0.014833066860834757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,64,0.0030346666773160298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,64,0.014661332964897156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,7168,32,0.002995199958483378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,7168,32,0.014826666315396628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,1536,0.009656533598899841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,2560,0.012733866771062216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,65536,0.28037548065185547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,65536,0.15868159929911296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,51200,0.12857813040415447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,51200,0.2214325269063314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,16384,0.08868052959442138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,16384,0.053158398469289145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,1024,0.008227199812730153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,12288,0.05836799939473471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,768,0.006311466793219249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,12288,0.043161598841349284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,10240,0.049320534865061445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,10240,0.03811733325322469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,512,0.005815466741720835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,256,0.00528106689453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,8192,0.04021226565043132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,8192,0.03491199811299642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,7168,128,0.005121066669623057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,7168,0.03475840091705322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,7168,0.031973334153493245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,6144,0.03039039969444275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,6144,0.02935466567675273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,65536,0.18183253606160482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,5120,0.029309866825739543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,5120,0.02701440056165059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,51200,0.13780694007873534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,4096,0.02172373334566752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,4096,0.024999467531840007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,16384,0.04705493450164795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,3584,0.019654399156570433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,3584,0.02329813241958618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,12288,0.03614826599756877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,3072,0.01699519952138265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,3072,0.021897600094477335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,10240,0.03147520025571187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,2560,0.014524799585342408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,2560,0.020658133427302043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,8192,0.025845332940419512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,2048,0.012489599982897441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,7168,0.023413334290186563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,2048,0.019463467597961425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,1536,0.010121599833170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,1536,0.018361600240071614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,6144,0.021074134111404418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,1024,0.007563733557860057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,1024,0.015726932883262636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,5120,0.018532266219456993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,768,0.006497066716353099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,768,0.015229866902033488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,4096,0.015596800049146018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,512,0.0039818666875362395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,512,0.015209600329399109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,3584,0.014829867084821067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,256,0.003446399917205175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,256,0.01492693324883779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,3072,0.013081600268681845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,2048,0.01023573378721873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,128,0.003091199944416682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,1024,0.007065600156784058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,128,0.014739200472831726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,64,0.0030026666820049284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,64,0.014829867084821067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,6144,32,0.002996266633272171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,6144,32,0.014620799819628397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,1536,0.009342933694521587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,2560,0.012039466698964437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,65536,0.23499840100606284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,65536,0.1415829340616862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,51200,0.1850378672281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,51200,0.10995413462320965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,16384,0.06406506697336832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,16384,0.04654506842295329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,768,0.00621973325808843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,12288,0.049211732546488446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,12288,0.037988265355428055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,512,0.00550186683734258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,10240,0.04203519821166992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,10240,0.03426239887873332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,256,0.005108266572157542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,8192,0.03426986535390218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,8192,0.030267733335494994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,7168,0.02839999993642171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,7168,0.028424533208211263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,6144,128,0.004915200173854828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,6144,0.02540160020192464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,6144,0.026408533255259197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,65536,0.1353098710378011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,5120,0.0214954674243927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,5120,0.024297600984573363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,51200,0.10241280396779377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,4096,0.02000746726989746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,4096,0.02281493345896403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,16384,0.036797865231831865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,3584,0.016033066312472026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,3584,0.021785600980122884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,12288,0.02941653331120809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,3072,0.014059733351071677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,3072,0.020770132541656494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,10240,0.02584213415781657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,2560,0.012195199728012085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,2560,0.01981546680132548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,8192,0.021474132935206093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,2048,0.01046399970849355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,2048,0.01872319976488749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,7168,0.0195413331190745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,1536,0.008634666601816814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,1536,0.016570666432380678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,6144,0.017705599466959633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,1024,0.0069013332327206925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,1024,0.01560640037059784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,5120,0.015829333662986757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,768,0.0053151999910672505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,768,0.015240533153216043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,4096,0.013223466277122498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,512,0.003701333453257879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,512,0.015169066190719605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,3072,0.011353600025177001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,256,0.003419733295838038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,256,0.014758400122324624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,3584,0.012432000041007996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,128,0.0030773334205150605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,128,0.014777599771817525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,2560,0.010588799913724262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,64,0.002856533229351044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,64,0.01464959979057312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,5120,32,0.0028768000503381092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,2048,0.00920960009098053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,5120,32,0.014679466684659323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,65536,0.11177813212076823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,65536,0.1870485305786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,51200,0.14788373311360675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,51200,0.09059733549753825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,1536,0.008426666259765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,16384,0.050639998912811277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,16384,0.03996906677881877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,1024,0.006140799820423126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,12288,0.040148266156514484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,12288,0.034281599521636966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,768,0.006077866752942403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,10240,0.03374293247858683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,256,0.005332266787687937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,512,0.005534933507442474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,8192,0.028130133946736652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,8192,0.027261867125829058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,5120,128,0.004991999765237173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,7168,0.025364265839258833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,7168,0.025672533114751178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,10240,0.03018346627553304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,6144,0.022600533564885457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,6144,0.024081067244211832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,65536,0.11683626969655354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,5120,0.017778132359186807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,5120,0.022574933369954427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,51200,0.09321813583374024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,4096,0.01667733391125997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,4096,0.021095466613769532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,16384,0.03363626797993978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,12288,0.02627413272857666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,3584,0.013187199831008911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,3584,0.020459733406702676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,10240,0.02325119972229004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,3072,0.011821867028872172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,3072,0.019785600900650024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,8192,0.01955839991569519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,2560,0.01035520037015279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,2560,0.01913706660270691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,7168,0.017867734034856163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,2048,0.008924800157546996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,2048,0.01814613342285156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,1536,0.007560533285140991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,1536,0.015528532862663268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,6144,0.015944533546765647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,1024,0.005827199916044871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,1024,0.015459199746449789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,5120,0.014177067081133523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,768,0.004048000027736028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,768,0.01527466674645742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,4096,0.01225279966990153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,512,0.0035573333501815797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,512,0.015051733454068503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,3584,0.011442133784294128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,256,0.0032149332265059153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,256,0.014755200346310934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,3072,0.01076693336168925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,128,0.003028266628583272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,128,0.01461013356844584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,2048,0.0086816002925237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,64,0.00277866671482722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,64,0.014538666605949402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,4096,32,0.00279573326309522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,4096,32,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,2560,0.010040533542633057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,65536,0.17117652893066407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,1024,0.006250666578610738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,65536,0.10448746681213379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,1536,0.007403733332951863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,51200,0.13530240058898926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,51200,0.08557973702748617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,16384,0.04489279985427856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,16384,0.037514666716257736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,12288,0.03515199820200603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,12288,0.03209386666615804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,768,0.005764266848564148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,10240,0.030343466997146608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,10240,0.028670932849248248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,512,0.005427200098832448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,8192,0.02528426647186279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,8192,0.025788799921671553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,256,0.0050784001747767125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,7168,0.022859734296798707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,7168,0.02462079922358195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,4096,128,0.00480320006608963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,6144,0.019053866465886436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,6144,0.023294933636983237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,65536,0.12001067002614338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,16384,0.033497599760691325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,51200,0.0939136028289795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,5120,0.01816320021947225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,4096,0.013351466258366904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,5120,0.02230613430341085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,4096,0.020483199755350748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,3584,0.012172800302505494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,3584,0.019638399283091225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,12288,0.026877866188685103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,3072,0.010956799983978272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,3072,0.0191648006439209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,10240,0.023502933979034423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,2560,0.009616000453631084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,2560,0.018717867136001588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,8192,0.01981226603190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,2048,0.008380800485610962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,2048,0.017106133699417114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,7168,0.018550399939219156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,1536,0.007314133147398631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,1536,0.015633066495259605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,6144,0.016450132926305136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,1024,0.005085866649945577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,1024,0.015345066785812378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,5120,0.014417066176732381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,768,0.0038592000802357995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,768,0.015266133348147073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,4096,0.012545067071914672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,512,0.0035861333211263022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,512,0.014882133404413859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,3072,0.010779733459154766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,256,0.0031850665807724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,256,0.014947199821472168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,3584,0.012051199873288471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,128,0.0029866665601730345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,128,0.014447999993960061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,2560,0.01018453339735667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,64,0.0028031999866167706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,64,0.014538666605949402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3584,32,0.002811733384927114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3584,32,0.01461120049158732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,2048,0.008964266379674275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,65536,0.1444277286529541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,65536,0.09127786954243979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,1536,0.007401599983374278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,1024,0.006426666676998138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,51200,0.11536320050557454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,51200,0.07387306690216064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,16384,0.04074346621831258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,16384,0.03538026809692383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,12288,0.032636799414952594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,768,0.005980800092220307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,12288,0.030423466364542646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,256,0.005171200136343638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,10240,0.02755519946416219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,10240,0.027730133136113482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,8192,0.023474133014678954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,8192,0.025352533658345538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,128,0.00488319993019104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,7168,0.021112533410390218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,7168,0.024220800399780272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3584,512,0.00560746689637502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,6144,0.01834453344345093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,6144,0.02273919979731242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,65536,0.1058463970820109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,5120,0.01616426706314087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,5120,0.021594667434692384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,51200,0.076036270459493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,4096,0.014097066720326743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,16384,0.030535467465718585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,4096,0.02001706759134928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,3584,0.012786133090655008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,3584,0.019371734062830607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,12288,0.024445867538452147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,3072,0.01007466713587443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,3072,0.019805866479873657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,10240,0.021348265806833903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,8192,0.017895466089248656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,2560,0.008942932883898417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,2560,0.017707733313242595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,2048,0.007906133433183034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,2048,0.01646080017089844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,7168,0.016289066274960837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,1536,0.006833066542943318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,1536,0.015915733575820924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,6144,0.014791466792424521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,1024,0.004316799839337667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,1024,0.01530880033969879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,5120,0.0132832000652949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,768,0.0038719999293486277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,768,0.015485866864522298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,4096,0.011723732948303223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,512,0.0035157332817713416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,512,0.014890666802724203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,3584,0.01035520037015279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,256,0.0032287999987602235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,256,0.014851199587186179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,3072,0.01002239982287089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,128,0.003049599876006444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,2048,0.008239999910195668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,128,0.014541866381963095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,64,0.002890666574239731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,64,0.014520532886187234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,3072,32,0.0029205332199732465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,3072,32,0.014589866995811463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,65536,0.122325332959493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,2560,0.009543466567993163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,65536,0.07870186964670817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,1536,0.007041066884994507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,1024,0.006296533346176148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,51200,0.06400320132573446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,51200,0.09750400384267172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,16384,0.03372693459192912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,16384,0.031335467100143434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,12288,0.02680640021959941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,12288,0.026846933364868163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,768,0.005746133128801982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,10240,0.023331199089686075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,10240,0.02505386670430501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,512,0.005454933146635691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,8192,0.019341866175333657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,8192,0.02322453260421753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,256,0.005029333134492239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,7168,0.017644800742467246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,7168,0.02241706649462382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,3072,128,0.004917333523432413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,6144,0.015779200196266174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,6144,0.021884800990422566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,65536,0.10284372965494792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,51200,0.08134506543477377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,5120,0.01376426617304484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,5120,0.020562134186426797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,4096,0.011868799726168316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,16384,0.02954346736272176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,4096,0.019477333625157675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,3584,0.0110944002866745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,3584,0.018959999084472656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,12288,0.02355626622835795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,3072,0.010281599561373393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,3072,0.01790613333384196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,10240,0.020552533864974975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,2560,0.008331733445326488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,8192,0.017615999778111777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,2560,0.016773333152135216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,7168,0.014910933375358582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,2048,0.007226666808128357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,2048,0.016267733772595723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,6144,0.014633599917093912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,1536,0.005862399935722351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,1536,0.01574186682701111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,1024,0.004216533402601878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,1024,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,5120,0.013011200229326883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,768,0.0038880000511805216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,768,0.015240533153216043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,4096,0.011453866958618164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,512,0.003815466662247976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,3072,0.009924266735712688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,512,0.015152000387509666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,256,0.003324799984693527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,256,0.014854400356610616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,3584,0.011058132847150166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,128,0.003020799905061722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,128,0.014588800072669984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,2560,0.009291733304659527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,64,0.0028949332733949023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,64,0.014419200023015341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,2048,0.007322666545708974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2560,32,0.002980266759792964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2560,32,0.014607999722162882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,65536,0.09708053270975749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,65536,0.06495893398920695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,51200,0.07647893428802491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,51200,0.054440534114837645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,1024,0.0061951999862988796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,16384,0.028113067150115967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,16384,0.028017065922419232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,1536,0.007001600166161854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,12288,0.022345600525538127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,12288,0.02444159984588623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,768,0.0057770664493242896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,10240,0.01949866612752279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,256,0.0050357331832249965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,10240,0.023196800549825033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,512,0.005335466563701629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,8192,0.021607466538747153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2560,128,0.0048767998814582825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,7168,0.014596266547838846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,7168,0.021179733673731486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,8192,0.01613653302192688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,6144,0.013031466801961263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,6144,0.020407466093699138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,65536,0.09519573052724203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,5120,0.011947733163833619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,5120,0.020553600788116456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,51200,0.0676522652308146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,4096,0.01035520037015279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,4096,0.01949440042177836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,16384,0.027420800924301148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,3584,0.009729066491127014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,12288,0.02188373406728109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,3584,0.018887466192245482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,3072,0.009093333284060161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,3072,0.018177066246668497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,10240,0.019475199778874717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,2560,0.008190933366616566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,8192,0.016516266266504924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,2560,0.017428267002105712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,7168,0.01514240006605784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,2048,0.0075103998184204105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,2048,0.016312533617019655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,1536,0.006167466441790262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,1536,0.01600320041179657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,6144,0.01381653348604838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,1024,0.004513066510359446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,1024,0.01573973298072815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,5120,0.012528000275293985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,768,0.004075733323891958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,768,0.015155200163523355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,4096,0.010526933272679647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,512,0.003585066646337509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,512,0.01474133332570394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,3072,0.00936853289604187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,256,0.0031850665807724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,256,0.014866133530934652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,1536,0.00674773355325063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,128,0.003033600002527237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,1024,0.005820799867312113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,128,0.014525866508483887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,2560,0.008392533659934998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,64,0.0028970666229724885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,2048,32,0.0028885332246621448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,64,0.014779733618100485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,2048,32,0.014735999703407287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,65536,0.07556800047556558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,65536,0.05542399883270264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,3584,0.010545066992441813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,51200,0.06024959882100424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,2048,0.007267199953397115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,51200,0.045978665351867676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,16384,0.02261120080947876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,16384,0.02476693391799927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,512,0.005272533496220907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,12288,0.018233599265416463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,12288,0.022612265745798745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,768,0.005508266886075338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,10240,0.01595626672108968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,10240,0.0214303990205129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,8192,0.013392000397046407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,8192,0.020218666394551596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,256,0.0048981333772341405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,7168,0.01241919994354248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,2048,128,0.0048320000370343525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,7168,0.02002453406651815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,6144,0.011386666695276897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,6144,0.020175999402999877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,65536,0.08484480381011963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,16384,0.025221333901087446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,5120,0.010240000486373902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,51200,0.06740480264027914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,5120,0.019208532571792603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,4096,0.009031466643015544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,4096,0.017863466342290243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,3584,0.008781866232554118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,3584,0.01725013256072998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,12288,0.02024959921836853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,3072,0.00782400021950404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,3072,0.01665066679318746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,10240,0.01791999936103821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,2560,0.006994133194287618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,8192,0.015254400173823037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,2560,0.016246400276819863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,2048,0.005835733314355215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,2048,0.015954132874806723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,7168,0.014097066720326743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,6144,0.012839466333389282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,1536,0.0048320000370343525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,1536,0.015770666797955833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,1024,0.004167466859022776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,1024,0.015229866902033488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,5120,0.011306666334470113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,4096,0.009544533491134644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,768,0.00382080003619194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,768,0.014923733472824097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,512,0.0035082665582497918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,3584,0.009173333644866943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,3072,0.008081066608428954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,256,0.0030730667213598887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,256,0.014517333110173544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,512,0.014818132917086283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,128,0.0029472000896930695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,128,0.014342400431632995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,2048,0.00692799985408783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,64,0.002721066772937775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,64,0.014408533771832785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1536,32,0.002791466563940048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1536,32,0.014450132846832275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,65536,0.05110613505045573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,65536,0.042676266034444174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,2560,0.007638399799664815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,51200,0.04102826515833537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,51200,0.036372268199920656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,1024,0.0059008002281188965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,16384,0.01649173299471537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,16384,0.022450133164723714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,1536,0.006524799764156342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,12288,0.013511466979980468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,12288,0.02033066749572754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,768,0.0054506664474805195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,10240,0.012177067001660664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,10240,0.01989439924558004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,512,0.005209599932034811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,8192,0.010518399874369304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,8192,0.0190720001856486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,256,0.00494400014479955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,7168,0.00995733340581258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,7168,0.01829973260561625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1536,128,0.004855466882387797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,6144,0.008923733234405517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,6144,0.017836799224217735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,65536,0.08464639981587728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,5120,0.008575999736785888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,51200,0.06682133674621582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,5120,0.017859200636545815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,4096,0.007810133198897044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,4096,0.017432532707850137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,16384,0.02516266703605652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,3584,0.007337599992752075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,3584,0.01710933248202006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,12288,0.02035306692123413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,3072,0.006675200164318084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,3072,0.016863999764124553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,10240,0.017845332622528076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,2560,0.006134399771690368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,2560,0.016578132907549538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,8192,0.01532266636689504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,2048,0.005422933399677277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,2048,0.016023466984430947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,7168,0.01385599970817566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,1536,0.004755199948946635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,1536,0.015372799833615622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,6144,0.012699733177820841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,5120,0.01123306651910146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,1024,0.00401706670721372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,1024,0.015092266599337259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,4096,0.009608532985051472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,768,0.0037791999677817024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,768,0.0151829332113266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,3072,0.008215466638406117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,512,0.003458133339881897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,3584,0.009115733702977498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,512,0.014973866939544677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,256,0.003067733347415924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,256,0.014708266655604044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,128,0.0028607999285062153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,128,0.014424533645311991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,2048,0.0069567998250325514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,64,0.0026634665826956432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,64,0.01446293294429779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,1024,32,0.0026549334327379864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,2560,0.007831466694672901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,1024,32,0.014418133099873862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,65536,0.04021226565043132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,65536,0.036804266770680744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,51200,0.032671999931335446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,51200,0.03229440053304036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,1536,0.00651093324025472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,16384,0.013593600193659464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,16384,0.02056106726328532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,1024,0.005820799867312113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,12288,0.011477333307266236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,12288,0.019987199703852335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,768,0.005595733225345611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,10240,0.010542933146158855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,10240,0.01881706714630127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,256,0.0049674664934476215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,8192,0.009491200248400371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,8192,0.018408532937367758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,512,0.005178666611512502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,1024,128,0.004825599988301595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,7168,0.01768959959348043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,7168,0.008711466193199157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,6144,0.008339200417200725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,6144,0.018067200978597008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,65536,0.08457280000050862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,5120,0.0074442664782206226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,5120,0.018484266599019368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,51200,0.06701227029164633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,4096,0.0072170664866765336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,4096,0.017813332875569663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,16384,0.025014400482177734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,3584,0.006822399795055389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,3584,0.016883200407028197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,12288,0.019975467522939046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,3072,0.00626986672480901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,3072,0.016709333658218382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,10240,0.017605332533518474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,2560,0.006145066519578298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,2560,0.016315733393033348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,8192,0.014669866363207499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,2048,0.005348266661167144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,2048,0.016105600198109946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,7168,0.01344000001748403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,1536,0.0047541335225105286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,1536,0.015406933426856995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,6144,0.011911466717720032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,1024,0.004016000032424927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,1024,0.015292800466219583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,5120,0.010600533088048298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,768,0.003685333331425985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,768,0.015222400426864624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,4096,0.009378133217493693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,512,0.003389866650104523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,512,0.014710399508476257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,3584,0.009018666545550029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,256,0.0030432000756263735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,256,0.014408533771832785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,3072,0.008078933258851369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,128,0.002850133428970973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,128,0.014371200402577718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,2048,0.0068810666600863145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,64,0.0026858667532602947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,64,0.014595199624697366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,768,32,0.002666666607062022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,768,32,0.014597333470980325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,65536,0.02871893246968587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,65536,0.031010133028030396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,1536,0.006559999783833821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,51200,0.023643734057744344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,51200,0.027219200134277345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,1024,0.005738666653633118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,16384,0.0107424000898997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,2560,0.007811200122038524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,16384,0.0194432000319163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,12288,0.009339732925097148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,12288,0.018875734011332194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,768,0.005471999943256378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,10240,0.008794666330019633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,10240,0.017845332622528076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,512,0.00517439991235733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,8192,0.008529067039489746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,8192,0.017882666985193887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,256,0.004958933095137278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,768,128,0.004827733337879181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,7168,0.007918933530648549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,7168,0.018175999323527016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,6144,0.00727040022611618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,6144,0.01798186699549357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,65536,0.08434453010559081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,5120,0.006545066833496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,51200,0.06723413467407227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,5120,0.01810773412386576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,4096,0.006109866499900818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,4096,0.01760639945665995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,16384,0.02490453322728475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,3584,0.006457599997520447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,3584,0.017108267545700072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,12288,0.019990400473276774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,3072,0.006202666461467743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,3072,0.016780799627304076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,10240,0.017417599757512413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,2560,0.006073600053787232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,2560,0.016636799772580466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,8192,0.014258133371671042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,2048,0.0053610667586326596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,2048,0.015927466750144958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,7168,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,1536,0.004796800017356872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,1536,0.015732266505559287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,6144,0.011848533153533935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,1024,0.004009599983692169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,1024,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,5120,0.010637866457303365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,768,0.003703466554482778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,768,0.015046399831771851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,4096,0.009346133470535279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,512,0.0033813332517941795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,512,0.015001599987347921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,3072,0.008086400230725606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,256,0.0030720000465710956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,256,0.014587733149528503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,3584,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,128,0.0029546665648619336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,128,0.014546133081118264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,2560,0.007732266684373219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,64,0.0027104000250498454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,64,0.014839466412862143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,512,32,0.002721066772937775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,2048,0.006888533135255177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,512,32,0.014872533082962037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,65536,0.01771946748097738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,65536,0.02442986567815145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,51200,0.014124799768129984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,51200,0.022830933332443237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,1536,0.0065194666385650635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,16384,0.007935999830563863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,16384,0.018339200814565023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,1024,0.005707733333110809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,12288,0.007520000139872233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,12288,0.017854932943979898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,512,0.0050794666012128195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,10240,0.006840533514817555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,10240,0.018101332585016887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,768,0.0055061335364977515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,8192,0.006379733482996623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,8192,0.017494400342305504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,256,0.00479360024134318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,7168,0.00619946668545405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,512,128,0.004712533454100291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,7168,0.01830186645189921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,6144,0.005925333499908448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,6144,0.018331732352574667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,65536,0.08414186636606852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,5120,0.006363733112812043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,5120,0.018283732732137046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,51200,0.0669696013132731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,4096,0.005992533266544342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,4096,0.017772799730300902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,16384,0.024000000953674317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,3584,0.006437333424886067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,3584,0.016924800475438435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,12288,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,3072,0.006142933170000712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,3072,0.01690559983253479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,10240,0.01673813263575236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,2560,0.006020266811052958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,2560,0.016424533724784852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,8192,0.014306132992108664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,2048,0.005392000079154968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,2048,0.015692800283432007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,7168,0.013176533579826354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,1536,0.0047199999292691554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,1536,0.015705600380897522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,6144,0.011780266960461933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,1024,0.004040533304214477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,1024,0.01514346698919932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,5120,0.010537599523862202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,768,0.003702399879693985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,768,0.01487573285897573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,4096,0.009224533041318258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,512,0.0033344000577926634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,512,0.015012266238530478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,3584,0.008922666311264038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,3072,0.008059733112653096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,256,0.002962133288383484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,256,0.014578133821487427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,2048,0.006909866631031036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,128,0.00278613343834877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,128,0.014422399799029031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,2560,0.007670400043328603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,1024,0.005805866420269012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,64,0.0026890667776266735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,256,32,0.002639999985694885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,64,0.014245333274205527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,256,32,0.014588800072669984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,65536,0.011384532848993937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,65536,0.021244800090789794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,51200,0.010290132959683736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,51200,0.020156800746917725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,1536,0.006427733103434245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,16384,0.006208000083764395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,16384,0.018129066626230875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,12288,0.006151466568311056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,12288,0.017800533771514894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,768,0.005493333439032236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,10240,0.006275199850400289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,10240,0.017588265736897788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,512,0.005130666494369507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,8192,0.006065066655476888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,8192,0.017571200927098594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,256,0.004839466512203216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,7168,0.006035199761390686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,7168,0.01844373345375061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,256,128,0.0046847999095916745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,6144,0.005980800092220307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,6144,0.017684266964594523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,65536,0.08380160331726075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,5120,0.006312533219655354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,5120,0.018175999323527016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,51200,0.0664469321568807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,4096,0.0059690664211908976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,16384,0.024120533466339113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,4096,0.017432532707850137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,3584,0.006356266637643178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,3584,0.01713493267695109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,12288,0.019107200702031455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,3072,0.006071466704209646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,3072,0.0166293332974116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,10240,0.016714666287104288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,2560,0.00602346658706665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,2560,0.016114133596420287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,8192,0.014135467012723288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,2048,0.005352533360322317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,2048,0.015833600362141927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,7168,0.013119999567667642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,6144,0.011852799852689107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,1536,0.0046847999095916745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,1536,0.01551040013631185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,1024,0.004117333392302195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,1024,0.01508799990018209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,5120,0.010506666700045268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,768,0.003692800054947535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,768,0.014871467153231302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,4096,0.009180800120035807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,512,0.003323733309904734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,512,0.014653866489728292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,3072,0.008061866462230682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,256,0.0032799998919169106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,256,0.014634666840235391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,3584,0.008854400118192036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,128,0.0028607999285062153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,128,0.014512000481287637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,2560,0.007660800218582153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,64,0.0025727999707063037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,64,0.014392532904942832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,128,32,0.0026186667382717133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,128,32,0.014206932981808982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,65536,0.008603733777999879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,65536,0.019436800479888917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,51200,0.007829333345095318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,51200,0.019774933656056724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,16384,0.005995733539263407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,16384,0.018068265914916993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,12288,0.006001066664854685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,12288,0.01744746764500936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,10240,0.0061749334136645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,2048,0.006855466465155284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,10240,0.017893334229787193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,8192,0.017270400126775106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,7168,0.005970133344332377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,7168,0.017866667111714682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,6144,0.005791999896367391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,6144,0.01741866668065389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,5120,0.006214400132497152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,5120,0.017876267433166504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,4096,0.005809066692988077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,4096,0.017334399620691936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,3584,0.006250666578610738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,3584,0.017064533631006875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,3072,0.005931733548641205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,3072,0.016432000199953715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,2560,0.005960533519585928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,1536,0.006374399860699971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,2560,0.01629973351955414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,2048,0.005261866748332978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,2048,0.01595200002193451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,1536,0.004642133414745331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,1536,0.015526400009791056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,1024,0.004013866682847341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,8192,0.005970133344332377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,1024,0.015413332978884378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,768,0.0036085332433382668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,512,0.0033312000334262846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,768,0.015025066335995993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,256,0.0029440000653266907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,512,0.014643200238545737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,128,0.0027882667879263563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,256,0.014586666226387024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,64,0.002621866762638092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,128,0.0144405335187912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,64,32,0.0026015999416510267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,64,0.014514133334159851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,65536,0.008381866415341695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,64,32,0.01442026694615682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,51200,0.00805866668621699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,65536,0.01972800095876058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,16384,0.006050133208433787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,51200,0.01952000061670939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,12288,0.005868799984455109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,16384,0.018119466304779053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,10240,0.006073600053787232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,12288,0.017489065726598106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,512,0.005070933202902476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,8192,0.0059690664211908976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,10240,0.0179967999458313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,7168,0.005873066683610281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,6144,0.005765333275000254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,7168,0.01801813244819641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,768,0.005409066875775656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,5120,0.006164266665776571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,6144,0.017492266496022542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,4096,0.005773866673310598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,5120,0.017798399925231932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,3584,0.00622080018122991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,4096,0.017197867234547935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,3072,0.0058794667323430385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,3584,0.016845866044362386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,2560,0.006006399790445963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,3072,0.01650879979133606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,256,0.004807466765244802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,2048,0.005368533233801523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,2560,0.016482133666674295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,1536,0.004666666686534882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,2048,0.015659733613332113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,1024,0.003947733342647553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,1536,0.01581653356552124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,128,0.004760533571243286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,768,0.003587199995915095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,1024,0.015244799852371215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,512,0.0033386667569478357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,256,0.00296426663796107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,768,0.01508799990018209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,128,0.0027850667635599775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,512,0.01467626690864563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,64,0.0026079999903837843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,17,32,32,0.002609066665172577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,128,0.014217600226402283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,64,0.01432213286558787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,32,0.014421332875887552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,17,128,1024,0.005737600227197012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,8192,0.017462400595347087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,16384,0.7495114644368489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,16384,0.39268372853597006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,17,32,256,0.014598400394121806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,12288,0.298526922861735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,12288,0.5666741053263347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,10240,0.4767871856689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,51200,1.1829205830891927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,10240,0.2507765293121338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,8192,0.3838752110799154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,8192,0.23946879704793295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,51200,2.3404576619466146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,7168,0.3328703880310059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,7168,0.18050773938496906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,6144,0.15762240091959637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,6144,0.28743254343668617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,5120,0.23955839474995932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,5120,0.1342506726582845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,16384,0.3575349489847819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,4096,0.192902406056722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,12288,0.270031992594401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,4096,0.1100490649541219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,51200,1.093948745727539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,3584,0.16856212615966798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,3584,0.11366399923960369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,8192,0.18228267033894857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,10240,0.2330911954243978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,3072,0.08773226737976074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,3072,0.17045440673828124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,2560,0.1218943993250529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,2560,0.07568853696187337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,2048,0.09766613642374675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,7168,0.18661227226257324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,6144,0.13874239921569825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,2048,0.07206400235493979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,1536,0.08761280377705892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,1536,0.05877439975738526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,5120,0.11681919892628986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,1024,0.05935786565144857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,1024,0.04007039864857991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,768,0.040098134676615396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,768,0.03446400165557861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,4096,0.09776746431986491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,512,0.03187413414319356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,512,0.027741867303848266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,3072,0.08327679634094239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,256,0.017771732807159425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,3584,0.08441493511199952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,256,0.021236266692479452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,128,0.00920746624469757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,128,0.018326399723688762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,2560,0.06258133252461752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,64,0.007131733496983846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,64,0.016382933656374613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,2048,0.05784746805826822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,32,0.0048320000370343525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,32,0.016541866461435954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,1024,0.02904319961865743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,1536,0.040446933110555014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,65536,1.212841542561849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,768,0.023371734221776328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,65536,2.3761632283528646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,51200,1.8720394134521485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,512,0.017401599884033205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,51200,0.9580277125040689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,16384,0.7080469131469727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,256,0.012757333119710288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,16384,0.370362663269043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,128,0.009556266665458679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,12288,0.515009053548177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,12288,0.24236373901367186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,10240,0.3862762769063314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,8192,0.3010133425394694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,8192,0.1660213311513265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,7168,0.25763093630472816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,7168,0.14646612803141276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,10240,0.20218772888183595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,6144,0.2315765380859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,6144,0.12690133253733318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,65536,1.1561205546061197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,5120,0.21629014015197753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,5120,0.10860693454742432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,51200,0.9074186960856119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,4096,0.17497812906901042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,16384,0.2969397226969401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,12288,0.22454187075297036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,4096,0.09093333085378011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,3584,0.1340501308441162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,3584,0.08104533354441325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,3072,0.11557226975758869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,10240,0.21746346155802407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,3072,0.07219733397165934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,2560,0.09745706717173258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,2560,0.062435201803843175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,8192,0.15122666358947753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,2048,0.08904853661855062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,2048,0.05329066514968872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,7168,0.13415466944376628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,1536,0.060644265015920004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,1536,0.04769173463185628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,6144,0.11618239879608154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,1024,0.041706665356953936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,1024,0.03512853384017944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,5120,0.09797866344451904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,768,0.036212265491485596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,768,0.02892799973487854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,4096,0.08423786958058675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,512,0.024768000841140746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,512,0.02413546641667684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,3072,0.0679253339767456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,256,0.012432000041007996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,256,0.019363200664520262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,3584,0.07079040209452311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,128,0.008551466464996337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,128,0.015634133418401083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,2048,0.043272535006205246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,64,0.004294399917125702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,2560,0.05255680084228516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,64,0.016105600198109946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,51200,32,0.016135467092196147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,1536,0.03445013364156087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,1024,0.025141332546869916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,65536,0.7429429372151692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,51200,32,0.004346666733423868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,51200,0.5837216059366862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,51200,0.2982080141703288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,16384,0.18498986562093098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,16384,0.11417919794718426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,768,0.02069973349571228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,65536,0.3773376146952311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,12288,0.14049386978149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,12288,0.09663253625233968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,512,0.015482667088508605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,256,0.010925867160161336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,10240,0.07264213562011719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,10240,0.11618133385976155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,51200,128,0.007971199850241344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,8192,0.0947872002919515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,8192,0.061861332257588705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,7168,0.08265173435211182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,7168,0.05618026653925577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,6144,0.0718826691309611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,6144,0.05060799916585287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,65536,0.3565802574157715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,5120,0.07060053348541259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,5120,0.044785066445668535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,51200,0.3146656036376953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,4096,0.05748586654663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,4096,0.03883093198140462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,16384,0.09484586715698243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,3584,0.047584001223246256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,3584,0.03633706569671631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,12288,0.0726645310719808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,3072,0.03806613286336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,3072,0.03292160034179688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,10240,0.06142186721165975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,2560,0.03218666712443034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,2560,0.030148265759150188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,8192,0.050171732902526855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,2048,0.02638826568921407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,2048,0.026958932479222614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,7168,0.04473813374837239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,1536,0.020947200059890748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,1536,0.023460266987482707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,6144,0.03935146729151408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,1024,0.014632532993952433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,1024,0.020771199464797975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,5120,0.03395413160324097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,768,0.011576533317565918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,768,0.01941759983698527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,4096,0.028060799837112425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,512,0.00874773363272349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,512,0.017479467391967773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,3072,0.022644267479578654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,256,0.005760000149408976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,256,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,3584,0.025641600290934246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,128,0.0035103999078273775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,128,0.014929067095120749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,2048,0.016749866803487144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,64,0.00327360009153684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,64,0.015147733688354491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,32,0.003328000009059906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,32,0.015176533659299215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,2560,0.020179200172424316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,1536,0.013745066523551942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,65536,0.2968693415323893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,65536,0.5637151718139648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,51200,0.23696533838907877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,51200,0.43624000549316405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,16384,0.1455349286397298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,1024,0.010780800382296245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,16384,0.08948480288187663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,12288,0.11125973065694172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,12288,0.06789653301239014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,768,0.009357866644859315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,10240,0.09353919823964438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,10240,0.059716268380482995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,512,0.007867733140786488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,8192,0.073089067141215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,256,0.005300266544024149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,8192,0.05021973450978597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,128,0.005005866785844167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,7168,0.06409706672032675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,7168,0.04604053497314453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,6144,0.054381867249806724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,6144,0.04233920176823934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,65536,0.28427947362264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,5120,0.052621865272521974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,5120,0.03736533323923747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,51200,0.2207029342651367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,16384,0.07596586545308431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,4096,0.04423893292744954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,4096,0.032945066690444946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,3584,0.0332202672958374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,3584,0.030640000104904176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,12288,0.05800319910049438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,3072,0.029469867547353108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,3072,0.0286954661210378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,10240,0.048979198932647704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,2560,0.02464639941851298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,2560,0.025964800516764325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,8192,0.04011413256327311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,7168,0.03662826617558797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,2048,0.021678932507832847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,2048,0.023463465770085654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,1536,0.016200533509254454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,6144,0.03184319933255513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,1536,0.021514666080474854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,1024,0.011724799871444702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,1024,0.01939520041147868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,5120,0.027643734216690065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,4096,0.026076799631118773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,3072,0.018845866123835243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,768,0.009821866949399311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,768,0.018339200814565023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,512,0.00747519979874293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,512,0.015473066767056783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,3584,0.021297067403793335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,256,0.003894400099913279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,256,0.015130666891733804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,1024,0.00946666697661082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,128,0.0035349334279696143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,128,0.014781866470972696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,2560,0.016797866423924765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,64,0.0032320000231266023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,64,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,32,0.0032810665667057036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,1536,0.01181653340657552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,32,0.01503679951032003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,65536,0.2616309324900309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,65536,0.48197228113810225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,2048,0.013703466455141703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,51200,0.3800138791402181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,51200,0.2036362648010254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,16384,0.144268798828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,16384,0.07775680224100748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,12288,0.0961184024810791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,12288,0.05959039926528931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,512,0.005551999807357788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,10240,0.08060906728108724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,768,0.00830506682395935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,10240,0.052198398113250735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,8192,0.06581759850184123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,8192,0.04474560022354126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,256,0.0051701332132021586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,7168,0.05327999989191691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,7168,0.04073280096054077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,128,0.004965333143870035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,6144,0.04673920075098674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,6144,0.037356801827748615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,65536,0.2394378662109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,5120,0.044819200038909913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,5120,0.03366506497065226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,51200,0.18029866218566895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,16384,0.06428053379058837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,4096,0.03685653209686279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,4096,0.03039466738700867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,3584,0.028406399488449096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,3584,0.027944533030192058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,12288,0.04939306577046712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,3072,0.025013333559036253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,3072,0.02595199942588806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,10240,0.041900801658630374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,8192,0.03508266607920329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,2560,0.021303466955820718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,2560,0.02387626568476359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,2048,0.01765120029449463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,2048,0.02210986614227295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,7168,0.03115839958190918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,1536,0.013771733641624451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,1536,0.020180267095565797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,6144,0.02773546576499939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,1024,0.010135466853777569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,1024,0.01842986742655436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,5120,0.02413653333981832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,768,0.008503466844558716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,768,0.016667733589808144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,4096,0.0204693337281545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,512,0.006750933329264323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,512,0.015144532918930054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,3584,0.019049600760142008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,256,0.003623466690381368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,256,0.014958932995796204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,3072,0.016683733463287352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,128,0.0032448001205921174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,128,0.014777599771817525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,2048,0.012654933333396911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,64,0.0030602666238943735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,64,0.014750933647155762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,32,0.0031040000418821974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,1024,0.008853333195050557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,32,0.014667733510335287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,2560,0.01458560029665629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,65536,0.20097920099894204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,1536,0.010940800110499065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,65536,0.3755274772644043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,51200,0.16109973589579266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,51200,0.29749011993408203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,16384,0.09260266621907552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,16384,0.061101865768432614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,12288,0.07084159851074219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,12288,0.050424532095591224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,768,0.0070720002055168155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,10240,0.05939093430836996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,10240,0.04452053308486938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,512,0.005399466554323832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,8192,0.0482367992401123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,8192,0.038312534491221115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,256,0.005153066913286845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,7168,0.042534399032592776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,7168,0.035488001505533856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,128,0.004829866687456766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,65536,0.1860479990641276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,51200,0.1463157335917155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,6144,0.043338668346405027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,6144,0.03530453443527222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,5120,0.03173120021820068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,5120,0.029556266466776532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,16384,0.051585066318511966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,4096,0.02580159902572632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,12288,0.0404362678527832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,4096,0.02658453385035197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,3584,0.023106133937835692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,3584,0.025265065828959148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,10240,0.03467626571655273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,3072,0.020577067136764528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,3072,0.023865600426991783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,8192,0.028775467475255327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,2560,0.01750719944636027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,2560,0.02198293407758077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,2048,0.014353066682815552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,2048,0.02058026591936747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,7168,0.025933865706125898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,1536,0.011617066462834676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,1536,0.019064533710479736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,6144,0.022957867383956908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,1024,0.008669867118199667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,1024,0.017948800325393678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,5120,0.02034026583035787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,768,0.007282133400440216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,768,0.01509119967619578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,4096,0.017230933904647826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,512,0.0056991999348004665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,512,0.0152319997549057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,3584,0.015703466534614564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,256,0.003621333340803782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,256,0.014816000064214071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,3072,0.013847466309865317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,128,0.0031680000325044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,128,0.014871467153231302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,128,0.004805333415667216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,2560,0.012552533547083536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,64,0.014684800306955972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,32,0.0030154667794704436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,32,0.014547200004259745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,2048,0.01088853379090627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,1536,0.009879466891288758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,65536,0.3287733395894369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,65536,0.18718825976053874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,1024,0.00788373351097107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,51200,0.25876053174336755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,51200,0.15078186988830566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,768,0.005876266459623972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,16384,0.10560746987660725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,512,0.0055178667108217875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,16384,0.05587946573893229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,64,0.0030858665704727173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,12288,0.06444266637166342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,12288,0.04618879954020182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,10240,0.056119465827941896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,10240,0.041354668140411374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,8192,0.043132801850636796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,8192,0.035846400260925296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,256,0.005129600067933401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,7168,0.038837333520253495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,7168,0.03374720017115275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,7168,0.02358186642328898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,6144,0.03335893154144287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,6144,0.030845866600672407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,5120,0.028943999608357744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,5120,0.027937066555023194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,65536,0.16469653447469074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,4096,0.026425600051879883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,4096,0.025099732478459674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,51200,0.13027520179748536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,16384,0.04612480004628499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,3584,0.023804799715677897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,12288,0.03658026854197184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,3584,0.023986132939656575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,3072,0.018657066424687705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,10240,0.031267199913660684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,3072,0.022756266593933105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,2560,0.015936000148455302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,2560,0.021591466665267945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,2048,0.01322773297627767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,8192,0.02605866591135661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,2048,0.020270933707555137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,1536,0.01076586643854777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,1536,0.018861865997314452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,6144,0.02104746699333191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,1024,0.00812906672557195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,1024,0.016115199526151022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,5120,0.018422400951385497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,768,0.0068917334079742435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,768,0.014989866813023885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,4096,0.015433599551518759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,512,0.003987200061480204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,512,0.015005866686503092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,3072,0.012906666596730551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,2560,0.011800533533096314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,256,0.0033621333539485933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,256,0.014775466918945313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,2048,0.01043839951356252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,128,0.0031008000175158186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,128,0.014797866344451904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,1536,0.009386666615804036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,64,0.0030271999537944795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,64,0.014605866869290671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,32,0.0030432000756263735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,1024,0.006790400048096975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,32,0.014577066898345948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,768,0.005656533439954122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,65536,0.1601312001546224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,65536,0.28278398513793945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,3584,0.014349866906801859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,51200,0.22381013234456382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,16384,0.07637759844462076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,16384,0.05065279801686605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,512,0.005542399982611338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,51200,0.128438401222229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,12288,0.05832639932632446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,12288,0.04463040033976237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,10240,0.050010665257771814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,10240,0.03877333402633667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,256,0.005156266689300537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,8192,0.03954879840215047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,8192,0.03403626680374146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,7168,0.03444693485895793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,7168,0.03180586695671082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,128,0.004889599978923798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,6144,0.030355199178059893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,6144,0.029367466767628986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,65536,0.14493652979532878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,5120,0.02906559904416402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,5120,0.027134933074315387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,51200,0.11494719982147217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,16384,0.04087573289871216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,4096,0.02421226700146993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,4096,0.023941334088643393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,3584,0.018962132930755615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,3584,0.022886399428049722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,12288,0.03269439935684204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,3072,0.01683733264605204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,3072,0.02188053329785665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,10240,0.027959465980529785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,2560,0.01476479967435201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,2560,0.0212501327196757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,7168,0.0212501327196757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,2048,0.01251413325468699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,2048,0.019372800985972084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,6144,0.018973867098490395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,1536,0.010032000144322713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,1536,0.01845546762148539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,8192,0.023260800043741862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,5120,0.016456533471743265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,1024,0.007536000013351441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,1024,0.015286399920781454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,768,0.006366933385531108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,768,0.015222400426864624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,4096,0.013859200477600097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,512,0.003885866701602936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,512,0.01509119967619578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,3584,0.013375999530156455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,256,0.0033610666791598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,256,0.014708266655604044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,3072,0.012034133076667786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,128,0.0031413334111372627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,128,0.01441493332386017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,2560,0.011052800218264262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,64,0.0028746667007605235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,64,0.014513066411018372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,32,0.0029557332396507262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,32,0.014497066537539164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,2048,0.009706667065620423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,65536,0.24224534034729003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,65536,0.1401365280151367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,1536,0.00879146655400594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,51200,0.19178560574849446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,51200,0.10906133651733399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,1024,0.006172800064086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,16384,0.06404373248418173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,16384,0.04595520099004109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,12288,0.049135998884836836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,768,0.005487999816735586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,12288,0.037673600514729816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,10240,0.041900801658630374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,10240,0.033580799897511796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,512,0.0054400001962979635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,10240,0.02495680054028829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,8192,0.03404266834259033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,8192,0.030003199974695843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,7168,0.028292266527811687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,256,0.005115733544031779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,7168,0.028549333413441975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,6144,0.025043199459711712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,6144,0.02612053354581197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,128,0.004866133133570353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,5120,0.021306665738423665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,5120,0.024202666680018105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,65536,0.12588480313618977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,4096,0.020020266373952232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,51200,0.09971840381622314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,16384,0.03616426785786946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,12288,0.028830933570861816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,3584,0.016115199526151022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,4096,0.012782933314641318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,3584,0.021651200453440347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,3072,0.01409386694431305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,3072,0.02058346668879191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,8192,0.020584533611933388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,2560,0.012239999572436015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,2560,0.019989333550135293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,6144,0.017008000612258913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,2048,0.010373333096504211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,2048,0.01867093245188395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,5120,0.01474240024884542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,4096,0.02245546579360962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,1536,0.008637866377830506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,1536,0.017191465695699057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,1024,0.006806399921576183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,1024,0.015554133057594299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,7168,0.019139200448989868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,768,0.005500799914201101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,768,0.015177599589029946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,768,0.005681066711743673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,512,0.003602133442958196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,512,0.015117866794268289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,256,0.00329066663980484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,256,0.014779733618100485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,3584,0.012145066261291504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,128,0.0030346666773160298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,128,0.014653866489728292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,3072,0.011044266819953918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,64,0.0028213332096735638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,64,0.014619732896486918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,32,0.0028736000259717304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,32,0.01467626690864563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,2560,0.010236799716949463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,2048,0.00897279977798462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,65536,0.18915306727091472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,65536,0.10769813060760498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,65536,0.11137920220692951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,51200,0.14851412773132325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,51200,0.09029760360717773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,16384,0.039698131879170734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,16384,0.05399680137634277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,12288,0.03948053518931071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,1536,0.007638399799664815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,12288,0.03396906852722168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,10240,0.03358506759007772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,10240,0.030664533376693726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,1024,0.006153599917888641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,8192,0.027832533915837603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,8192,0.02785173257191976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,512,0.005389866729577383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,7168,0.02515733242034912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,7168,0.025649066766103106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,256,0.005128533144791921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,6144,0.02235413392384847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,6144,0.024583466847737632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,128,0.004894933104515076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,5120,0.01800853411356608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,5120,0.023036799828211465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,51200,0.08630293210347494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,16384,0.03139306704203288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,4096,0.015969066818555196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,4096,0.020968532562255858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,12288,0.025493333737055462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,3584,0.014507733782132468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,3584,0.02025493383407593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,3072,0.011943466464678447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,3072,0.01950826644897461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,10240,0.021950932343800862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,3072,0.010300800204277039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,2560,0.010453333457310993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,2560,0.01890773375829061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,2048,0.009051733215649923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,8192,0.018272000551223754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,2048,0.008323200047016144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,1536,0.007645866771539052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,1536,0.01588373382886251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,7168,0.01696746746699015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,1024,0.005976533393065134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,1024,0.015292800466219583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,1024,0.005784533421198527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,768,0.0038090666135152185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,768,0.015095466375350952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,6144,0.015171200037002563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,512,0.0035402665535608927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,512,0.015149866541226705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,512,0.005201066533724466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,256,0.003206400076548258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,2048,0.01767786741256714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,256,0.014871467153231302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,128,0.0029450667401154833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,128,0.01462399959564209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,5120,0.013215999801953635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,64,0.002869333326816559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,64,0.014756266276041666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,32,0.0029365333418051405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,32,0.01472106675306956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,4096,0.011665067076683045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,3584,0.011241599917411804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,65536,0.10411946773529053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,2560,0.009642666578292847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,51200,0.13649813334147137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,51200,0.08563093344370523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,1536,0.0068234667181968685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,16384,0.0450111985206604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,16384,0.03685973485310872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,768,0.0057205334305763245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,12288,0.03494826555252075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,12288,0.03218026757240296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,65536,0.1704576015472412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,10240,0.03430080016454061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,10240,0.028947200377782183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,256,0.00506986677646637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,8192,0.025026132663091023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,8192,0.026273065805435182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,128,0.004820266862710317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,7168,0.022802132368087768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,7168,0.02454293370246887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,6144,0.020438400904337566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,65536,0.10388800303141277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,6144,0.023361066977183022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,5120,0.017591466506322227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,5120,0.02179626623789469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,51200,0.07922666867574056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,4096,0.014909866452217101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,16384,0.03023680051167806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,4096,0.02077546715736389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,3584,0.013252266248067222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,3584,0.019810134172439577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,12288,0.024322134256362916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,3072,0.01104639967282613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,3072,0.019266132513682047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,10240,0.021088000138600668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,2560,0.009724799791971843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,2560,0.018552533785502114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,8192,0.017767467101415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,2048,0.008414933085441589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,2048,0.01646613379319509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,7168,0.016379732886950174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,1536,0.007325866818428039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,1536,0.015558399756749473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,6144,0.014651733636856078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,1024,0.0049781332413355505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,1024,0.015289599696795145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,5120,0.013158399860064188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,768,0.0038922667503356934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,768,0.015269333124160766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,4096,0.011533866326014202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,512,0.0035818666219711304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,512,0.014916266997655234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,3584,0.010713600118954976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,256,0.0031968000034491217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,256,0.01439786652723948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,3072,0.010073600212732951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,128,0.002924799919128418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,128,0.014453333616256715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,2560,0.00958293378353119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,64,0.0028181334336598715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,64,0.014636799693107605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,2048,0.007665066421031952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,32,0.0028661333024501802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,32,0.014596266547838846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,65536,0.14565760294596355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,65536,0.09056959946950277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,1536,0.006805333495140076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,51200,0.11531306902567547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,51200,0.07372693220774332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,16384,0.040294400850931805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,16384,0.035807998975118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,1024,0.005986133217811584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,12288,0.032051199674606325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,12288,0.029895466566085816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,768,0.005592533449331919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,10240,0.027405865987141925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,10240,0.02754559914271037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,512,0.005426133175690969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,8192,0.0233514666557312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,8192,0.025124265750249224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,256,0.005109333495299021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,7168,0.0208512008190155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,7168,0.024066134293874105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,128,0.004839466512203216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,6144,0.01821440060933431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,6144,0.022617600361506143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,6144,0.013994666934013366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,5120,0.01576533317565918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,5120,0.020988800128300986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,65536,0.09527146816253662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,4096,0.014733866850535075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,4096,0.02016426722208659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,51200,0.07631680170694986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,3584,0.012667733430862426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,3584,0.019335466623306274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,12288,0.022541866699854533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,3072,0.01149120032787323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,3072,0.018547199169794717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,10240,0.01973653237024943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,16384,0.028249599536259967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,2560,0.009113599856694538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,2560,0.0173909326394399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,8192,0.016760534048080443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,2560,0.008618666728337606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,2048,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,2048,0.01641493340333303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,1536,0.006715733309586842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,1536,0.01583573321501414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,7168,0.014805333813031516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,1024,0.00448639988899231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,1024,0.015090133746465048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,5120,0.012458667159080505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,768,0.003806933263937632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,768,0.015006933609644571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,4096,0.01106346646944682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,512,0.0035317334036032355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,3584,0.010570666193962098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,512,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,512,0.005376000205675761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,256,0.0031829332311948144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,256,0.014684800306955972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,128,0.003005866706371307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,128,0.014500266313552857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,3072,0.009687466422716777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,64,0.0027232001225153605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,64,0.014413866400718688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,32,0.0028736000259717304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,32,0.01437226633230845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,65536,0.12215147018432618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,65536,0.07812906901041666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,51200,0.09727040131886801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,51200,0.06343679825464885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,2048,0.007351466516653697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,16384,0.03381653229395549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,16384,0.031787733236948654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,1536,0.006668800115585327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,16384,0.027559467156728107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,12288,0.02671146591504415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,1024,0.005854933460553487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,12288,0.02738453348477681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,10240,0.023176532983779908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,10240,0.02518080075581868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,8192,0.019433599710464478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,8192,0.023235199848810832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,768,0.005545599758625031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,7168,0.0175327996412913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,7168,0.021972266832987468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,256,0.004964266717433929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,6144,0.015576533476511636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,128,0.004917333523432413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,6144,0.021385600169499717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,5120,0.013610666990280152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,5120,0.0206389327843984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,65536,0.09336640040079752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,4096,0.011788800358772278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,4096,0.01928960084915161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,51200,0.07421120007832846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,3584,0.011037866274515789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,3584,0.018759467204411826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,12288,0.02188906669616699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,3072,0.010231467088063557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,10240,0.01806933283805847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,3072,0.017620267470677693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,2560,0.008392533659934998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,2560,0.016246400276819863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,8192,0.01648319959640503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,2048,0.0073290665944417315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,2048,0.016132266322771708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,7168,0.015090133746465048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,1536,0.006093866626421611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,1536,0.015593600273132325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,6144,0.013794133067131042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,1024,0.0041461333632469176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,1024,0.015251200397809347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,5120,0.01251413325468699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,768,0.0038453333079814913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,768,0.015240533153216043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,4096,0.010855467120806376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,512,0.003457066665093104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,512,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,3584,0.010360532999038696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,256,0.0031839999059836066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,256,0.014654933412869772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,3072,0.0086517333984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,128,0.0028394666810830434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,128,0.014364799857139588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,2560,0.008242133259773254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,64,0.0027562665442625684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,64,0.014619732896486918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,32,0.002828799933195114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,2048,0.00729066679875056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,32,0.014603733023007711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,1536,0.006658133367697399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,65536,0.09744853178660075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,65536,0.06477333307266235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,51200,0.07636160055796305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,51200,0.05429226557413737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,16384,0.02775040070215861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,16384,0.027727999289830524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,1024,0.0060810665289560955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,768,0.005549866457780202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,12288,0.022325332959493002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,12288,0.024282666047414143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,512,0.005277866621812185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,256,0.005086933573087057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,10240,0.019383466243743895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,10240,0.0228767991065979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,8192,0.01595946649710337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,8192,0.021617066860198975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,128,0.0048767998814582825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,7168,0.014503467082977294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,7168,0.020757333437601725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,65536,0.08543999989827475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,6144,0.013066666324933371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,6144,0.02040106654167175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,51200,0.06846293608347574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,5120,0.011735467116038005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,5120,0.02027946710586548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,4096,0.01027733286221822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,4096,0.01877546707789103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,16384,0.025240532557169598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,3584,0.009645866354306538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,3584,0.017911465962727864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,12288,0.020526933670043945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,3072,0.008980266253153483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,3072,0.017026132345199584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,10240,0.017961599429448447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,2560,0.007618133227030437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,8192,0.015482667088508605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,2560,0.015988266468048094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,2048,0.006569600105285645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,2048,0.015997866789499916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,7168,0.014102400342623393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,1536,0.005313066641489664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,1536,0.015527466932932535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,6144,0.013005866607030233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,1024,0.004145066688458125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,1024,0.015186132987340293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,5120,0.011716266473134358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,768,0.0037759999434153237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,768,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,4096,0.010381866494814556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,512,0.0033962666988372804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,512,0.014999467134475707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,3584,0.009477333227793375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,256,0.00306986669699351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,256,0.01472106675306956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,3072,0.008825600147247314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,128,0.002903466671705246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,128,0.014664533734321594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,2560,0.007712000111738841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,64,0.002752000093460083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,64,0.014418133099873862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,32,0.002746666719516118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,2048,0.006964266796906789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,32,0.014409599701563516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,65536,0.055320533116658535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,65536,0.0753653367360433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,51200,0.06002560059229532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,51200,0.04595626592636108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,1536,0.0065984000762303666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,16384,0.022385066747665404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,16384,0.024851200977961223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,1024,0.005857066810131073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,12288,0.018177066246668497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,12288,0.022184532880783082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,768,0.005448533097902933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,10240,0.015785599748293556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,10240,0.02193386753400167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,512,0.005206400156021118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,8192,0.013499733805656434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,256,0.0049098665515581764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,8192,0.020246400435765585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,7168,0.012433066964149475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,7168,0.020001065731048585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,7168,0.014190933108329773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,6144,0.011217066645622253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,6144,0.019629865884780884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,128,0.0046965335806210835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,5120,0.010282666484514872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,5120,0.01821119983990987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,5120,0.011212799946467083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,4096,0.009124267101287841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,4096,0.017107200622558594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,65536,0.08393599987030029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,3584,0.008700799942016602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,16384,0.025102933247884113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,8192,0.015173332889874777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,6144,0.012988799810409546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,3584,0.016572800278663636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,3072,0.008131200075149536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,2560,0.006695466736952464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,3072,0.01688106656074524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,2560,0.016116266449292503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,2048,0.005752533177534739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,12288,0.020081067085266115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,2048,0.015828266739845276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,1536,0.004769066472848257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,1536,0.015434666474660238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,10240,0.01768853267033895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,4096,0.009878399968147277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,1024,0.004063999901215235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,1024,0.015211733182271323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,51200,0.06717759768168131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,768,0.0036960000793139136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,768,0.01504746675491333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,3584,0.009110400080680847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,512,0.003390933324893316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,2560,0.007867733140786488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,512,0.014818132917086283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,256,0.0030794667700926462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,256,0.01454080045223236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,2048,0.006884266436100006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,256,0.0049898669123649595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,128,0.0029365333418051405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,128,0.014519466956456503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,128,0.004869333406289419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,64,0.002739199995994568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,32,0.0027690666417280836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,64,0.014417066176732381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,32,0.014403200149536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,65536,0.05039360125859579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,51200,0.04082026481628418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,65536,0.042138667901357015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,65536,0.08376533190409342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,16384,0.01592639982700348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,1024,0.005781333148479462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,16384,0.021767467260360718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,12288,0.013427199920018515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,12288,0.020172800620396933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,768,0.005469866593678792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,10240,0.012138666709264119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,10240,0.019756799936294554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,3072,0.008131200075149536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,8192,0.010547199845314026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,8192,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,1536,0.006619733572006225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,7168,0.009841066598892213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,7168,0.018607999881108603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,51200,0.03642026583353679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,6144,0.00885973374048869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,6144,0.017797333002090455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,6144,0.012391466895739238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,5120,0.008521599570910136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,5120,0.017961599429448447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,512,0.005133866767088572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,4096,0.007595733304818471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,4096,0.017709867159525553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,51200,0.06722666422526041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,3584,0.007226666808128357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,3584,0.017672532796859743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,12288,0.01983786622683207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,3072,0.006491733094056447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,3072,0.016386133432388306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,10240,0.01770240068435669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,2560,0.0060917332768440245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,2560,0.016238933801651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,8192,0.01492693324883779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,2048,0.00544213354587555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,2048,0.0158869336048762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,7168,0.013736533125241599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,1536,0.004790399968624115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,16384,0.024936532974243163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,1536,0.016157866517702738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,1024,0.004045866678158442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,1024,0.015388799707094827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,5120,0.01056106686592102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,768,0.003748266647259394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,768,0.0150709331035614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,4096,0.009406933188438415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,512,0.0034506666163603462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,512,0.014844800035158793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,3584,0.008955732981363932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,256,0.003067733347415924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,256,0.014455466469128927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,3072,0.008205866813659668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,128,0.002794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,128,0.014571733276049294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,2560,0.007730133334795634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,64,0.002669866631428401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,2048,0.006915199756622315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,32,0.002749866743882497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,64,0.014444800217946372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,32,0.01427733302116394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,65536,0.03977173169453939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,65536,0.03653973340988159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,51200,0.03255680004755656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,51200,0.03218453327814738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,1536,0.006593066453933716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,16384,0.01350933313369751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,1024,0.005745066702365876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,16384,0.020632533232371013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,12288,0.011527466773986816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,12288,0.01898026665051778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,768,0.005529599885145823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,10240,0.01035520037015279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,10240,0.01886826753616333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,512,0.005251200000445048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,8192,0.009377066294352214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,256,0.005017599960168203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,8192,0.01784320076306661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,128,0.0047765334447224935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,7168,0.008687999844551087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,7168,0.017754666010538735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,65536,0.08424746990203857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,6144,0.007826133569081625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,6144,0.017948800325393678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,51200,0.06670400301615396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,6144,0.01183146635691325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,5120,0.007586133480072021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,5120,0.018184532721837364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,4096,0.0063733334342638654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,4096,0.017364267508188883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,16384,0.02495786746342977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,3584,0.0067114666104316715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,3584,0.017046399911244712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,12288,0.020006400346755982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,3072,0.0063178668419520065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,3072,0.016769067446390788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,10240,0.01738133430480957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,2560,0.006144000093142191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,2560,0.016743467251459757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,2048,0.005393066505591074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,2048,0.01615466674168905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,2048,0.006938666601975759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,1536,0.0046965335806210835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,1536,0.015595733126004537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,8192,0.014847999811172486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,1024,0.004008533308903376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,1024,0.015241600076357522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,7168,0.013241599996884665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,768,0.0036650667587916053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,768,0.015147733688354491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,5120,0.010544000069300334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,512,0.003332266708215078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,512,0.014935466647148132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,4096,0.009248000383377076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,256,0.0030464000999927522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,256,0.014491732915242514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,3584,0.008959999680519104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,128,0.0027872001131375628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,3072,0.008062933385372163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,128,0.014390400052070618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,64,0.0026602665583292644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,64,0.014340266585350037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,32,0.0026464000344276427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,32,0.01439573367436727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,65536,0.02835306723912557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,2560,0.007656533519426982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,65536,0.030406399567921953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,51200,0.023437867561976113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,51200,0.026332799593607587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,1536,0.006455466647942861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,16384,0.010685867071151734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,16384,0.019550933440526327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,1024,0.005843199789524078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,12288,0.009278933207194012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,12288,0.01731946667035421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,768,0.005505066613356272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,10240,0.008756267031033833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,10240,0.018129066626230875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,512,0.00505920002857844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,8192,0.007851733267307282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,8192,0.01808746655782064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,256,0.004929066697756449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,7168,0.006828799843788147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,7168,0.0180949330329895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,128,0.0048437332113583885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,6144,0.006250666578610738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,6144,0.018310399850209554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,65536,0.08358720143636068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,5120,0.0064074665307998655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,5120,0.01795733372370402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,5120,0.010502400000890096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,51200,0.06706133683522543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,4096,0.005997866888840993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,4096,0.01760960022608439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,3584,0.006540800134340922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,3584,0.017012266318003337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,16384,0.024727465709050496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,3072,0.006189866860707601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,3072,0.01662613352139791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,12288,0.01953173279762268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,2560,0.006083199878533682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,2560,0.016472533345222473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,10240,0.017207467555999757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,2048,0.005416533350944519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,2048,0.015887999534606935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,8192,0.014186666409174601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,1536,0.004748799900213877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,1536,0.015497600038846334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,1536,0.006470400094985962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,1024,0.003988266736268997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,7168,0.013015466928482055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,1024,0.015247999628384908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,768,0.003618133316437403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,768,0.015010133385658264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,6144,0.01183573305606842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,512,0.003332266708215078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,4096,0.009412266810735067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,512,0.014613333344459533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,3584,0.008941866954167684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,256,0.0029834667841593427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,256,0.014722133676211039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,3072,0.008200533191363017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,128,0.002872533351182938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,128,0.014342400431632995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,2560,0.007758933305740357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,64,0.0026506667335828146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,64,0.014363732933998109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,2048,0.006926933427651723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,32,0.002703999976317088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,32,0.01454080045223236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,65536,0.02417280077934265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,1024,0.00576853354771932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,51200,0.014603733023007711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,768,0.005355733136336008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,16384,0.007468800246715546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,65536,0.018203733364741008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,16384,0.018023467063903807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,12288,0.0063733334342638654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,12288,0.017459199825922648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,512,0.005128533144791921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,10240,0.006346666812896728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,10240,0.017953066031138103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,256,0.00494400014479955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,8192,0.006162133316198985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,8192,0.01770346760749817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,51200,0.022341332832972207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,7168,0.006071466704209646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,7168,0.01809813380241394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,128,0.004791466891765595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,6144,0.005958400170008342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,6144,0.017663999398549398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,65536,0.08334293365478515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,5120,0.006357333560784657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,5120,0.018224000930786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,51200,0.06625920136769613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,4096,0.005946666498978933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,4096,0.017199999094009398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,16384,0.024016000827153525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,3584,0.006411733229955037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,12288,0.019012266397476198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,10240,0.016556800405184428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,3072,0.0060138667623202005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,8192,0.014246400197347006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,3072,0.016381866733233132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,2560,0.006085333228111267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,3072,0.008004266520341237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,2560,0.016174933314323424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,2048,0.005381333331267038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,2048,0.01578133304913839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,7168,0.013033599654833476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,2048,0.006915199756622315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,1536,0.004717866579691568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,1536,0.015369600057601929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,1024,0.0040778666734695435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,1024,0.01499626636505127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,6144,0.011854933698972066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,768,0.00363520011305809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,768,0.014947199821472168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,3584,0.016860800981521606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,512,0.003398400048414866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,512,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,5120,0.01053013304869334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,256,0.003091199944416682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,256,0.014667733510335287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,4096,0.009212799866994222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,128,0.002773333340883255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,128,0.014418133099873862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,3584,0.008929066856702169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,64,0.0027146667242050173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,64,0.01439573367436727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,32,0.0027232001225153605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,32,0.01444586714108785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,65536,0.011901866396268208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,65536,0.021919999519983926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,2560,0.007618133227030437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,51200,0.01050453285376231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,51200,0.01959786613782247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,1536,0.006472533444563548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,16384,0.006118399898211161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,16384,0.018360533316930137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,1024,0.005824000140031179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,12288,0.006253866851329804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,12288,0.017602133750915527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,768,0.005478399991989136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,10240,0.006337066491444905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,10240,0.017810134092966716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,512,0.005156266689300537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,8192,0.006134399771690368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,8192,0.017372800906499227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,256,0.004819199939568837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,7168,0.006011733412742614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,7168,0.018078933159510292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,128,0.0047199999292691554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,6144,0.005707733333110809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,6144,0.017463467518488564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,65536,0.08352746963500976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,51200,0.06607893308003744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,5120,0.006056533257166544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,5120,0.01797119975090027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,16384,0.02403200070063273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,4096,0.005734399954477946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,4096,0.017199999094009398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,12288,0.01904319922129313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,3584,0.006180266539255777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,3584,0.016618667046229045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,10240,0.016727467377980552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,3072,0.005851733187834421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,3072,0.016681599617004394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,8192,0.01421333352724711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,2560,0.005762133498986562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,2560,0.016200533509254454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,2560,0.007518933216730754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,2048,0.0051925331354141235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,2048,0.01581653356552124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,7168,0.01304213305314382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,1536,0.0047189335028330484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,1536,0.015498666961987814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,6144,0.011730133493741354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,1024,0.00395413339138031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,1024,0.01509119967619578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,768,0.003651199986537298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,768,0.015016532937685647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,5120,0.010446932911872864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,512,0.0033130665620168054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,512,0.01464959979057312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,4096,0.009219200412432352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,256,0.0029088000456492106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,256,0.014599466323852539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,3584,0.008870399991671244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,128,0.002850133428970973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,128,0.01423679987589518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,3072,0.008161066472530365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,64,0.002674133330583572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,64,0.014320000012715658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,32,0.0026869334280490874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,32,0.01430506706237793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,65536,0.008807466427485148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,65536,0.0195360004901886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,2048,0.006866133213043213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,51200,0.00774186650911967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,16384,0.00597866674264272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,51200,0.019223467508951823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,16384,0.01768959959348043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,12288,0.006007466713587443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,10240,0.006132266422112783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,12288,0.017570134003957114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,8192,0.0059349333246548975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,1536,0.006366933385531108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,10240,0.017735467354456583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,7168,0.005779199798901876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,6144,0.0056533331672350565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,7168,0.017756799856821694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,5120,0.006098133325576782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,6144,0.01761386593182882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,5120,0.018093866109848023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,4096,0.017192532618840538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,3584,0.006142933170000712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,1024,0.005678933362166087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,3584,0.01691626707712809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,3072,0.005730133255322774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,3072,0.0164192001024882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,2560,0.005650133391221364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,2048,0.004984533290068308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,2560,0.015896532932917276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,1536,0.0044490665197372435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,2048,0.01636693378289541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,1024,0.003752533346414566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,1536,0.015212800105412802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,768,0.003532800078392029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,1024,0.015079466501871744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,512,0.0031648000081380212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,768,0.014708266655604044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,256,0.002940800040960312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,512,0.014628266294797262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,128,0.002700799951950709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,256,0.014298666516939798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,64,0.002619733413060506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,128,0.014226133624712626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,32,0.0026176000634829206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,64,0.014260266224543253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,32,0.014111999670664468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,65536,0.007195733487606049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,51200,0.007086933155854543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,65536,0.020988800128300986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,16384,0.006131199995676676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,51200,0.01911466717720032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,12288,0.005787733197212219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,8192,0.017347200711568197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,10240,0.005964800218741099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,12288,0.01762239933013916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,8192,0.005936000247796377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,10240,0.017723733186721803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,768,0.005342933535575867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,7168,0.0057322666049003605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,8192,0.01735146641731262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,4096,0.0056309332450230915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,7168,0.017862399419148765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,6144,0.017417599757512413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,5120,0.005949866771697998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,512,0.004993066688378652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,4096,0.005644799768924713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,5120,0.017619200547536216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,3584,0.005989333490530649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,4096,0.017234132687250773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,3072,0.005739733576774597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,3584,0.01685333251953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,2560,0.005685333410898844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,3072,0.016748799880345663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,2048,0.005014400184154511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,2560,0.01609173317750295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,1536,0.004440533121426901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,2048,0.015824000040690102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,1024,0.003868799904982249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,1536,0.015480533242225647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,1024,0.015072000026702882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,768,0.0035690667728583017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,256,0.004806399842103322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,512,0.0032778667906920114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,256,0.0029418667157491045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,768,0.014754133423169455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,128,0.002796799937884013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,256,0.01461120049158732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,64,0.0026015999416510267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,128,0.01420799990495046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,32,0.0025898667673269907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,64,0.014267733693122864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,32,0.014141866564750671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,128,0.004811733464399974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,16384,0.017912532885869345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,6144,0.005645866692066193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,16384,0.7488501230875652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,16384,0.3934783935546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,512,0.014705066879590353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,12288,0.29822187423706054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,12288,0.2702272097269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,51200,1.1817291259765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,10240,0.25046720504760744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,10240,0.47353601455688477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,8192,0.20653227170308433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,8192,0.3788373311360677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,51200,2.339922078450521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,12288,0.5672042846679688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,7168,0.18086506525675455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,7168,0.3370911916097005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,6144,0.15797866185506185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,6144,0.3200885454813639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,5120,0.23768746058146156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,5120,0.1435530662536621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,4096,0.19412585894266765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,16384,0.3588063875834147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,4096,0.1099285364151001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,3584,0.16853973070780437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,3584,0.09971946875254313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,51200,1.0970784505208333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,3072,0.17010025978088378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,10240,0.2643818696339925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,3072,0.0997920036315918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,2560,0.12174293200174968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,8192,0.18179413477579753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,2560,0.07518400351206461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,2048,0.09776746431986491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,2048,0.06323519945144654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,7168,0.1864576021830241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,6144,0.1385759989420573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,1536,0.08746559619903564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,1536,0.052861865361531576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,1024,0.0512223998705546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,1024,0.040232535203297934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,5120,0.11719573338826497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,768,0.040009601910909014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,768,0.03457599878311157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,4096,0.10890773137410481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,512,0.028882133960723876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,512,0.02749333381652832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,3584,0.08409173488616943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,256,0.015830399592717488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,256,0.021270400285720824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,3072,0.07792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,128,0.01025493343671163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,128,0.018227199713389076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,2560,0.0655082662900289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,64,0.006983466446399689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,64,0.01641706625620524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,65536,32,0.004649599889914194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,65536,32,0.016436266899108886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,2048,0.05118293364842733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,1536,0.04486613273620606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,1024,0.0291701336701711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,768,0.02332586646080017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,512,0.017426133155822754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,256,0.012199466427167256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,65536,1.212317911783854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,65536,128,0.009427199761072796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,51200,0.9573258717854818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,16384,0.36951894760131837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,16384,0.6148650487263997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,51200,1.8715413411458333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,65536,2.3665631612141924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,12288,0.24120853741963705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,12288,0.46998294194539386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,10240,0.20187093416849772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,10240,0.4494016011555989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,8192,0.3056405385335287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,8192,0.18891520500183107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,7168,0.2574570655822754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,7168,0.14687466621398926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,65536,1.1591573079427084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,6144,0.26057279904683434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,51200,0.9065301259358725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,16384,0.297380256652832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,6144,0.13305173714955648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,5120,0.18818880716959635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,12288,0.2604885260264079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,5120,0.10929386615753174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,4096,0.09068586826324462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,4096,0.17921919822692872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,10240,0.20866026878356933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,3584,0.15358826319376628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,3584,0.08244053522745767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,3072,0.11349120140075683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,8192,0.15162347157796224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,3072,0.07194986343383789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,2560,0.06312640110651652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,2560,0.11082346439361572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,2048,0.07815786997477213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,2048,0.05325119892756144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,2048,0.04310719966888428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,1536,0.05880959828694662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,1536,0.04376213153203328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,6144,0.13361600240071614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,1024,0.04645119905471802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,5120,0.10506986776987712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,1024,0.0370410680770874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,768,0.03172266681989034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,768,0.029306666056315107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,4096,0.07965973218282064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,512,0.024851200977961223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,512,0.02432960073153178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,3584,0.07097813288370768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,256,0.013698133826255798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,256,0.019595734278361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,3072,0.06940480073293051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,128,0.007865599791208903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,128,0.016103466351826988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,2560,0.05266666809717814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,64,0.004362666606903076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,64,0.016083199779192606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,7168,0.13347519238789876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,51200,32,0.004314666489760081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,51200,32,0.015602133671442666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,1536,0.03446933428446452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,65536,0.3763968149820963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,65536,0.7419818878173828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,51200,0.29932053883870446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,51200,0.583245849609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,1024,0.025593600670496625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,16384,0.21773653030395507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,768,0.020204800367355346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,16384,0.1075050671895345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,12288,0.0851370652516683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,512,0.015237333377202353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,10240,0.11612906455993652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,256,0.011313066879908244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,10240,0.07301653226216634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,8192,0.09433279832204183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,51200,128,0.008329600095748901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,8192,0.061810131867726645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,7168,0.08269226551055908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,7168,0.056195199489593506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,12288,0.140065066019694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,6144,0.07216320037841797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,6144,0.050388268629709874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,6144,0.0392522652943929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,5120,0.0603498657544454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,5120,0.044725334644317626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,65536,0.35669228235880535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,4096,0.057396264870961514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,4096,0.03887253204981486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,4096,0.02805440028508504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,3584,0.04346133470535278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,3584,0.03617493311564128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,51200,0.279695987701416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,3072,0.04385706583658854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,16384,0.09502613544464111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,3072,0.03345706860224406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,12288,0.07260053157806397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,2560,0.032331732908884685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,2560,0.0302239994208018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,2048,0.02637653350830078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,2048,0.026792534192403156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,10240,0.061383465925852455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,1536,0.02074346741040548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,1536,0.023601067066192628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,8192,0.050399998823801674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,1024,0.014663466811180114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,1024,0.020454400777816774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,7168,0.04486506779988607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,5120,0.033930667241414386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,768,0.01167466640472412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,768,0.019271467129389444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,3584,0.0255786657333374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,512,0.008728532989819845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,512,0.017499732971191406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,512,0.0074442664782206226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,256,0.005735466877619425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,256,0.015309866269429526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,3072,0.02542613347371419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,128,0.003457066665093104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,128,0.015105066696802774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,2560,0.01981653372446696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,64,0.0032330666979153953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,64,0.015035733580589294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,2048,0.01660586694876353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,16384,32,0.003260799994071325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,16384,32,0.015122133493423461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,1536,0.013689600427945457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,65536,0.2967125256856283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,65536,0.5550367991129558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,1024,0.01062506635983785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,16384,0.14581333796183268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,51200,0.23643093109130858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,16384,0.08936213652292888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,51200,0.4418688138326009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,12288,0.11173226833343505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,12288,0.07623786926269531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,10240,0.09378453095753989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,768,0.009305600325266521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,10240,0.06023573478062948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,8192,0.07637333075205485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,8192,0.05020266771316528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,8192,0.0400607983271281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,7168,0.06651093165079752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,7168,0.04593706528345744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,256,0.005421866476535797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,6144,0.05564800103505453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,16384,128,0.004981333514054617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,6144,0.04143893321355184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,5120,0.04563093185424805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,5120,0.03726826508839925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,65536,0.2847061475118001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,4096,0.043178665637969973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,4096,0.03277013301849365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,51200,0.2271455923716227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,4096,0.02333013415336609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,3584,0.030825599034627275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,3584,0.037868801752726236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,16384,0.07590933640797934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,3072,0.029346134265263873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,12288,0.05796800057093302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,3072,0.02813226580619812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,2560,0.025006934007008867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,2560,0.025591466824213666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,2560,0.016616533199946083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,2048,0.023785599072774253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,10240,0.049030399322509764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,1536,0.016235733032226564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,1536,0.021821866432825722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,7168,0.035843201478322345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,1024,0.012945066889127096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,1024,0.019128533204396565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,6144,0.03205759922663371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,768,0.009781333804130554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,768,0.018284799655278523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,2048,0.02037866711616516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,512,0.0076117331782976794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,512,0.015278933445612588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,5120,0.027907200654347736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,256,0.004130133241415024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,256,0.015081600348154704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,3584,0.021314134200414024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,128,0.0035413332283496858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,3072,0.01882773240407308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,128,0.01498133341471354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,64,0.0032426667710145317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,64,0.014920533696810404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,12288,32,0.003180799881617228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,12288,32,0.014842666188875833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,2048,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,1536,0.011819733182589214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,65536,0.2611519972483317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,65536,0.48079681396484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,1024,0.009481599926948548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,51200,0.2039573351542155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,51200,0.37953920364379884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,51200,0.17993280092875164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,768,0.00823359986146291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,16384,0.12925013701121013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,16384,0.08646079699198404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,12288,0.0956981341044108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,512,0.005580799778302511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,12288,0.05976426601409912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,10240,0.07907413641611735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,10240,0.05158400138219198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,10240,0.04190186659495036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,8192,0.06562879880269369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,256,0.005284266670544943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,8192,0.044455464680989584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,7168,0.05319466590881348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,7168,0.04082346757253011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,6144,0.046426665782928464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,6144,0.037565867106119796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,12288,128,0.00490880012512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,5120,0.03923306862513225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,5120,0.033768534660339355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,65536,0.23953065872192383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,4096,0.036345601081848145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,4096,0.030085333188374835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,16384,0.06408746639887491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,3584,0.02948906620343526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,3584,0.02853226661682129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,12288,0.04955093463261922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,3072,0.02499413291613261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,3072,0.026114133993784587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,8192,0.035122132301330565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,2560,0.021280000607172646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,7168,0.031141332785288495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,2560,0.02446720004081726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,2048,0.01757226586341858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,2048,0.022117332617441813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,6144,0.0278165340423584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,1536,0.013802666465441385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,1536,0.020242132743199668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,1024,0.010170666376749675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,1024,0.0189194659392039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,1024,0.008834133545557659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,768,0.008412800232569377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,768,0.01684266726175944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,5120,0.02412266731262207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,512,0.006743466854095459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,512,0.015117866794268289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,4096,0.020824533700942994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,256,0.003623466690381368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,256,0.015081600348154704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,3584,0.018582399686177573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,128,0.003294933338960012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,128,0.014757333199183145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,3072,0.016609066724777223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,64,0.0030879999200503034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,64,0.014758400122324624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,2560,0.014712533354759217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,2048,0.012724266449610392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,10240,32,0.014765866597493491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,1536,0.010910933216412861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,65536,0.20085546175638833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,65536,0.37594559987386067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,51200,0.29736852645874023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,51200,0.16117332776387533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,16384,0.09913492997487386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,768,0.007180800040562947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,16384,0.06101226806640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,512,0.005562666555245717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,10240,32,0.0030847998956839246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,12288,0.0708896001180013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,12288,0.050560001532236734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,10240,0.05914346774419149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,10240,0.04452586571375529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,8192,0.048097066084543866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,10240,0.034414935111999514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,8192,0.03843626578648885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,7168,0.04261866807937622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,7168,0.035651199022928876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,7168,0.025869866212209065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,6144,0.037628801663716634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,6144,0.032654933134714764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,256,0.005118933320045471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,5120,0.0315615991751353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,5120,0.02991360028584798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,10240,128,0.004916266600290934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,4096,0.02595413327217102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,4096,0.026394667228062947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,4096,0.017119999726613364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,3584,0.023256532351175942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,3584,0.02545493245124817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,65536,0.1865045388539632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,3072,0.023277866840362548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,3072,0.023715200026830037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,51200,0.14609386126200358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,2560,0.019879466295242308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,16384,0.051680000623067227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,2560,0.02224853237469991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,2048,0.014349866906801859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,12288,0.04041066567103068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,2048,0.02068906625111898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,1536,0.011582932869593303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,1536,0.019323732455571493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,8192,0.028893866141637164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,1024,0.0086218665043513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,1024,0.01741546591122945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,1024,0.007761066655317943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,768,0.0073525334397951765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,768,0.015373866756757101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,6144,0.02300800085067749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,512,0.005776000022888183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,512,0.015094400445620219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,5120,0.020111999909083047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,256,0.00363520011305809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,3584,0.015528532862663268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,256,0.014913066228230795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,3072,0.013769599795341491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,128,0.003219199925661087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,128,0.01464959979057312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,2560,0.012593066692352295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,64,0.0030720000465710956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,64,0.014685866236686707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,8192,32,0.003115733216206233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,1536,0.009867733716964722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,8192,32,0.014566399653752646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,65536,0.18541119893391927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,768,0.0059690664211908976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,65536,0.326908810933431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,512,0.005592533449331919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,65536,0.1647007942199707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,51200,0.15062506993611652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,51200,0.2582261403401693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,16384,0.05790826479593912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,16384,0.09246400197347006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,12288,0.06312640110651652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,12288,0.04624319871266683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,10240,0.05572266578674316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,10240,0.040909866491953537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,10240,0.031125332911809283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,2048,0.011060266693433126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,8192,0.04554239908854167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,8192,0.025913600126902265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,7168,0.037964800993601486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,7168,0.03360533316930135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,6144,0.033633065223693845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,6144,0.031202133496602374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,256,0.005048533280690512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,5120,0.02904319961865743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,5120,0.027898667256037395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,5120,0.01831573247909546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,4096,0.023578667640686037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,4096,0.025164800882339477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,8192,128,0.00484799991051356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,3584,0.02109973430633545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,8192,0.03601599931716919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,3584,0.024167466163635253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,3072,0.018517333269119262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,3072,0.022946133216222128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,3072,0.012760532895723977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,2560,0.01598186691602071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,2560,0.0213045338789622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,51200,0.13025386333465577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,2048,0.014920533696810404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,16384,0.05302506685256958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,2048,0.020027732849121092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,1536,0.010814932982126872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,12288,0.03640106519063314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,1536,0.018609066804250084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,1024,0.008144000172615051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,1024,0.015492266416549683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,7168,0.023601067066192628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,768,0.006920533378918965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,768,0.015128533045450846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,6144,0.02095466653505961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,512,0.004422399898370107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,512,0.015315199891726175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,4096,0.01548479994138082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,256,0.0033919999996821085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,256,0.01509119967619578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,3584,0.014251733819643656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,128,0.003067733347415924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,128,0.014728533228238425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,2560,0.011994666854540507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,64,0.0029535998900731405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,64,0.014545067151387533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,7168,32,0.0029258665939172106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,7168,32,0.014738133549690247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,65536,0.28201068242390953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,2048,0.010231467088063557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,1536,0.009130666653315227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,1024,0.006810666620731353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,51200,0.22365973790486654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,51200,0.12806719938913982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,768,0.0059349333246548975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,16384,0.05153813362121582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,16384,0.07586560249328614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,12288,0.05830613374710083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,512,0.00551146666208903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,12288,0.043005867799123125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,10240,0.050040535132090246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,10240,0.03797866503397624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,256,0.005087999999523163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,65536,0.15932159423828124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,8192,0.039476267496744794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,8192,0.036659200986226395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,7168,0.03473386764526367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,7168,0.03201280037562053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,7168,0.021078399817148843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,7168,128,0.004884266853332519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,6144,0.03031040032704671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,6144,0.02909333308537801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,5120,0.025462400913238526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,16384,0.04103893438975016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,5120,0.026984532674153645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,65536,0.1439125378926595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,4096,0.02418559988339742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,4096,0.02441813349723816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,4096,0.014075733224550881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,3584,0.01928000052769979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,3584,0.02334933280944824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,51200,0.11489706834157307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,3072,0.019040000438690186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,3072,0.022029866774876915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,12288,0.03263253370920817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,3072,0.011795199910799662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,2560,0.014706133802731832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,2560,0.020951465765635172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,2048,0.012488533059755962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,2048,0.019397334257761637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,10240,0.027934932708740236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,1536,0.00998293360074361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,1536,0.018501333395640054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,8192,0.023219199975331624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,1024,0.00755626658598582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,1024,0.015927466750144958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,1024,0.006100266675154368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,768,0.006409599880377452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,768,0.015198933084805808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,768,0.00561706672112147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,512,0.0038218667109807336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,6144,0.01904746691385905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,512,0.015128533045450846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,256,0.0032981333633263906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,256,0.014633599917093912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,5120,0.0166293332974116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,128,0.0030346666773160298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,128,0.014637866616249084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,64,0.0028575999041398365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,64,0.014546133081118264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,6144,32,0.0029493334392706556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,6144,32,0.014502400159835815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,65536,0.24235413869222006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,65536,0.13966399828592937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,3584,0.01318186620871226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,51200,0.1917535940806071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,2560,0.010959999759991963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,2048,0.009718400239944459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,51200,0.10862186749776204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,51200,0.09983680248260499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,16384,0.06368000109990438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,16384,0.03609600067138672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,12288,0.04926400184631348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,16384,0.045762133598327634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,1536,0.00876800020535787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,10240,0.04187306563059489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,12288,0.03794453144073486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,8192,0.03393599987030029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,10240,0.03404479821523031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,8192,0.030403200785319013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,7168,0.02824746568997701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,7168,0.028420267502466838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,512,0.005413333574930826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,6144,0.024984532594680788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,6144,0.026899200677871705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,128,0.004775466521581014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,5120,0.02132586638132731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,5120,0.024359466632207234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,5120,0.014837333559989929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,4096,0.01774719953536987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,4096,0.022426666816075642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,65536,0.12469866275787353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,6144,256,0.005063466727733612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,3584,0.015955199797948204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,3584,0.02168533404668172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,3072,0.014057599504788718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,3072,0.020398932695388793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,3072,0.01100373367468516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,2560,0.012241066495577494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,2560,0.019693867365519205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,12288,0.028780800104141236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,2048,0.010328533252080281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,10240,0.024778666098912557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,2048,0.019160532951354982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,1536,0.00869546631971995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,8192,0.020595200856526694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,1536,0.01738133430480957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,1024,0.006760533154010773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,1024,0.015352533260981242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,7168,0.01906879941622416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,768,0.005245866874853769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,768,0.015326933066050211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,6144,0.01699519952138265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,512,0.0037205333511034647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,512,0.015144532918930054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,4096,0.012957866986592612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,256,0.0032479998966058097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,1536,0.007438933352629344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,256,0.01479573349157969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,256,0.005172266562779745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,128,0.0030378667016824085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,128,0.01458026667435964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,64,0.002922666569550832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,64,0.014553599556287131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,5120,32,0.002914133419593175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,3584,0.012085333466529846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,5120,32,0.014588800072669984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,65536,0.185915740331014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,2560,0.010326400399208069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,65536,0.1109013319015503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,51200,0.14718292554219564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,2048,0.008985599875450135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,51200,0.09017493724822997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,16384,0.050589867432912196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,51200,0.08629120190938314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,16384,0.03970453341801961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,1024,0.0062047998110453285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,12288,0.039656531810760495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,12288,0.03411733309427897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,10240,0.033676799138387045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,768,0.005670399963855743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,10240,0.03094826738039653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,8192,0.02805440028508504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,8192,0.027883734305699664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,512,0.005446400245030721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,7168,0.02521386742591858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,7168,0.026139734188715617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,5120,128,0.00487253318230311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,6144,0.022425599892934165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,6144,0.02433919906616211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,6144,0.015080533425013223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,5120,0.01953279972076416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,5120,0.022614399592081703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,4096,0.01604693333307902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,4096,0.021580799420674642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,65536,0.10673387050628662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,3584,0.016169599692026772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,3584,0.020424532890319824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,3584,0.011188266674677531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,3072,0.01204800009727478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,3072,0.019428267081578573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,16384,0.03146880070368449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,2560,0.010494933525721232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,2560,0.018926932414372762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,12288,0.025525333484013875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,2048,0.008949333429336548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,2048,0.01803413430849711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,10240,0.021895466248194377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,1536,0.007749333480993907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,1536,0.015406933426856995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,8192,0.018273067474365235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,1024,0.005746133128801982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,1024,0.015564800302187601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,7168,0.01686933239301046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,768,0.003972266614437103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,768,0.015258666872978211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,5120,0.013424000144004822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,512,0.0035594666997591654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,512,0.014940800269444785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,4096,0.011673600474993388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,256,0.0032810665667057036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,256,0.01469013293584188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,3072,0.010174933075904845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,128,0.0029535998900731405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,128,0.014363732933998109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,2560,0.009674666325251262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,64,0.0027797333896160126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,64,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,2048,0.008312533299128216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,4096,32,0.002889599899450938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,4096,32,0.014696533481280008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,1536,0.006981333096822103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,65536,0.17035519282023112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,1024,0.005859200159708659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,65536,0.10466240247090657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,51200,0.08548053105672201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,51200,0.13461333910624187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,16384,0.04879573186238607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,16384,0.037623465061187744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,12288,0.03486613432566325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,12288,0.031850665807724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,768,0.005578666428724925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,10240,0.030125866333643597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,10240,0.02898240089416504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,512,0.005334400137265523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,8192,0.024898133675257363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,8192,0.026065067450205488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,256,0.005042133231957754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,8192,0.017959467569986978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,7168,0.022693334023157756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,7168,0.024613332748413087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,6144,0.020345600446065266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,6144,0.023849600553512575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,6144,0.014865066607793173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,5120,0.01753386656443278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,5120,0.022318933407465616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,4096,128,0.004770133395989736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,4096,0.014818132917086283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,4096,0.02070186734199524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,65536,0.10355947017669678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,51200,0.08253546555836996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,3584,0.013229866822560629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,3584,0.02021546761194865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,3072,0.010840533177057902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,16384,0.03044266700744629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,3072,0.01986453334490458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,12288,0.024094933271408082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,2560,0.009687466422716777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,2560,0.018691200017929076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,2560,0.009459199508031209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,2048,0.008410666386286418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,2048,0.01700693368911743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,10240,0.021207465728123983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,1536,0.0071370666225751235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,1536,0.01597760021686554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,7168,0.016370133558909098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,1024,0.004769066472848257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,1024,0.015473066767056783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,5120,0.013182933131853739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,768,0.003885866701602936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,768,0.01513706644376119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,4096,0.01113813320795695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,512,0.0035487999518712364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,512,0.014813866217931113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,3584,0.011147733529408772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,256,0.0031839999059836066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,3072,0.010041600465774536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,256,0.01478506624698639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,128,0.0029045333464940387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,128,0.01458133359750112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,64,0.0027424000203609467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,64,0.014653866489728292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3584,32,0.002898133297761281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3584,32,0.014550399780273438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,2048,0.008039466540018718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,65536,0.1459338665008545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,65536,0.09112106959025065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,51200,0.1151157299677531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,1536,0.006742399930953979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,51200,0.07346133391062418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,16384,0.04041386842727661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,16384,0.03534293174743652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,1024,0.006109866499900818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,12288,0.03142613371213277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,12288,0.02969599962234497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,768,0.005613866448402405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,10240,0.027218133211135864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,10240,0.02733760078748067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,512,0.005310933291912079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,8192,0.023178666830062866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,8192,0.025865600506464644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,256,0.0051146666208903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,7168,0.02095466653505961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,7168,0.024627200762430825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3584,128,0.004801066716512045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,6144,0.018542933464050292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,6144,0.02308053374290466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,65536,0.09543573061625163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,5120,0.016160000363985697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,5120,0.021127466360727945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,51200,0.07603306770324707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,4096,0.014043733477592468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,4096,0.020247467358907065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,16384,0.028024532397588092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,3584,0.012716799974441528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,3584,0.019393066565195717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,12288,0.02265920042991638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,3072,0.011180800199508668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,3072,0.018759467204411826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,10240,0.019631999731063842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,2560,0.009110400080680847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,2560,0.017240534226099648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,8192,0.015970133741696677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,2048,0.007815466821193695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,2048,0.015999999642372132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,7168,0.015412267049153647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,1536,0.006804266571998596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,1536,0.015389866630236306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,6144,0.014034133156140646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,1024,0.004440533121426901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,1024,0.015387733777364096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,5120,0.012601600090662638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,768,0.003929600119590759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,768,0.015147733688354491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,4096,0.011024000247319539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,512,0.003505066782236099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,3584,0.010674132903416952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,512,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,256,0.0031786667803923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,256,0.014885333180427552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,3072,0.00969493289788564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,128,0.003014400104681651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,128,0.014408533771832785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,2560,0.00846613347530365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,64,0.002845866729815801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,64,0.014595199624697366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,3072,32,0.0028394666810830434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,3072,32,0.01477226714293162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,2048,0.0070709332823753355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,65536,0.1215722640355428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,65536,0.07851306597391763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,51200,0.09692800045013428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,51200,0.06322559912999472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,1536,0.006595199803511302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,16384,0.033769599596659344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,1024,0.006088533500830332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,16384,0.03123413324356079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,768,0.005514666438102722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,12288,0.026781866947809856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,12288,0.026975999275843304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,512,0.005206400156021118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,10240,0.02331733306248983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,10240,0.024922666947046916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,256,0.004971733192602793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,8192,0.019495467344919838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,8192,0.02328426639238993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,8192,0.016376533110936484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,7168,0.017566933234532674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,7168,0.022318933407465616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,3072,128,0.005014400184154511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,6144,0.015526400009791056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,6144,0.02132373253504435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,65536,0.0932522694269816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,5120,0.01362666686375936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,5120,0.020517333348592123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,51200,0.06924053033192953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,4096,0.011806933085123698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,4096,0.01906026601791382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,16384,0.02752853234608968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,3584,0.011026133100191753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,3584,0.018759467204411826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,3072,0.01027733286221822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,3072,0.01753600041071574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,12288,0.022046933571497597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,2560,0.008363733688990276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,2560,0.01685546636581421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,10240,0.01912213365236918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,2048,0.00726506660381953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,2048,0.015989333391189575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,7168,0.015013333161671957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,1536,0.005836800237496694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,6144,0.013680000106493631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,1536,0.01564906636873881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,1024,0.004217599829037985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,1024,0.015431466698646545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,5120,0.012313600381215413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,768,0.003902933249870936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,768,0.015117866794268289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,4096,0.010497066378593444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,512,0.0034901333351929987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,512,0.014803199966748556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,3584,0.010549333691596986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,256,0.0031637333333492277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,1536,0.006622933348019918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,256,0.014626133441925048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,128,0.002930133293072383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,3072,0.009008000294367473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,128,0.014600533246994018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,64,0.0027744000156720476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,64,0.014510933558146158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2560,32,0.0027935999135176343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2560,32,0.014588800072669984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,65536,0.0972597360610962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,65536,0.06438399950663248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,2560,0.00811839997768402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,51200,0.07589226563771566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,2048,0.007159466544787089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,51200,0.053511468569437656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,16384,0.027829333146413164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,16384,0.027525333563486735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,16384,0.02553173303604126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,12288,0.022350933154424033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,12288,0.024151466290156045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,1024,0.005915733178456625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,10240,0.01943146586418152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,10240,0.023371734221776328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,10240,0.01812480092048645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,8192,0.01609599987665812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,8192,0.021721599499384563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,768,0.005585066477457682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,7168,0.014516266187032065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,7168,0.02073813279469808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,512,0.00522986650466919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,6144,0.013017599781354269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,6144,0.020258132616678873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,256,0.005115733544031779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,5120,0.011808000008265178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2560,128,0.00481279989083608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,5120,0.019808000326156615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,4096,0.010307199756304423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,4096,0.0190175990263621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,65536,0.0856725295384725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,3584,0.009621333082516987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,3584,0.017768534024556477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,51200,0.06838613351186117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,3072,0.00897920032342275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,3072,0.01630400021870931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,3072,0.008571733037630718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,2560,0.007674666742483775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,2560,0.01628266672293345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,12288,0.020510933796564736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,2560,0.007835733393828075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,2048,0.00676800012588501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,2048,0.006949333349863689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,2048,0.016101333498954772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,1536,0.00480320006608963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,1536,0.015596800049146018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,8192,0.015598932902018229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,1024,0.004088533421357473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,1024,0.015441067020098367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,7168,0.014362667004267374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,768,0.0037813333173592886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,768,0.014842666188875833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,768,0.005465599894523621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,512,0.003401600072781245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,512,0.014805333813031516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,6144,0.012802132964134216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,256,0.0030794667700926462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,256,0.014574933052062988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,5120,0.011781332890192668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,128,0.002916266769170761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,128,0.014459733168284097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,4096,0.010195199648539226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,64,0.0027285332481066385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,64,0.014442666371663412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,2048,32,0.0027232001225153605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,2048,32,0.014450132846832275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,65536,0.07502826849619547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,65536,0.05454933245976766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,3584,0.009452799956003826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,51200,0.0598634680112203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,51200,0.04589333136876424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,1536,0.006559999783833821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,16384,0.022530132532119752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,16384,0.024714666604995727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,12288,0.018221867084503175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,12288,0.022371200720469157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,1024,0.005830400188763936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,10240,0.015795200069745382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,10240,0.021647999684015908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,10240,0.017896533012390137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,8192,0.013364266355832419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,8192,0.020677334070205687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,256,0.004936533172925314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,7168,0.012295466661453248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,7168,0.01983786622683207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,128,0.004779733220736186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,6144,0.011201066772143047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,6144,0.019367466370264687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,65536,0.08454079627990722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,5120,0.010055466492970785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,5120,0.019243733088175455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,2048,512,0.005190399785836538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,4096,0.009160533547401428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,51200,0.06739947001139322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,4096,0.017256534099578856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,3584,0.00862613320350647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,3584,0.017033600807189943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,16384,0.02512213389078776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,3072,0.008186666667461396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,3072,0.016736000776290894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,12288,0.020241065820058187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,2560,0.007102933526039123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,2560,0.016416000326474507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,8192,0.01530026694138845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,2048,0.0054506664474805195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,2048,0.015995732943216958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,7168,0.014003200332323709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,1536,0.004741333425045013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,1536,0.015637333194414772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,6144,0.012713600198427835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,1024,0.00402453343073527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,1024,0.015465600291887918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,5120,0.011342933773994446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,768,0.003718400001525879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,768,0.014908799529075622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,4096,0.009505066275596618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,512,0.003337600082159042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,512,0.014905599753061929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,512,0.005086933573087057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,3584,0.009124267101287841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,256,0.014562132954597472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,3072,0.008071466783682505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,128,0.002808533360560735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,128,0.014666666587193808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,2560,0.007686399916807811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,64,0.0026389333109060925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,64,0.0145087997118632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,32,0.0026538667579491934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1536,32,0.014506666858990987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,65536,0.050595200061798094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,65536,0.0420415997505188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1536,256,0.0031178665657838186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,51200,0.04081173340479533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,51200,0.03622080087661743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,2048,0.006903466582298279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,16384,0.016054399808247886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,1536,0.006541866560777028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,16384,0.022205867369969687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,12288,0.013453867038091025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,1024,0.0057770664493242896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,768,0.005532800157864889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,10240,0.012235732873280843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,10240,0.019824000199635823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,256,0.004901333153247834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,12288,0.020121600230534872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,8192,0.010573866963386535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,8192,0.019287467002868652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,8192,0.01516480048497518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,7168,0.009920000036557516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1536,128,0.00481279989083608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,7168,0.018403200308481853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,6144,0.008797867099444072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,6144,0.017825067043304443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,65536,0.08434879779815674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,5120,0.00842133363087972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,5120,0.018634666999181114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,51200,0.067031462987264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,4096,0.007922133306662242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,4096,0.01762239933013916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,4096,0.009290666381518046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,3584,0.0073088000218073535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,3584,0.01689493258794149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,12288,0.0201855997244517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,3072,0.006577066580454509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,3072,0.01726079980532328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,10240,0.017729065815607705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,2560,0.0060810665289560955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,2560,0.01620586713155111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,16384,0.024998400608698526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,2048,0.0053269331653912864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,2048,0.015921066204706825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,1536,0.004780800143877665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,1536,0.015410133202870688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,7168,0.013352533181508383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,1024,0.004093866546948751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,1024,0.015352533260981242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,6144,0.011912533640861511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,768,0.0036757332583268487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,768,0.015134933590888976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,5120,0.01050453285376231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,512,0.003432533393303553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,512,0.015000533064206442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,512,0.005110399921735128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,256,0.0030432000756263735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,256,0.014758400122324624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,3584,0.009030399719874065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,128,0.002811733384927114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,128,0.014579199751218162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,3072,0.00811839997768402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,64,0.00276053324341774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,64,0.01446399986743927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,1024,32,0.002669866631428401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,1024,32,0.014350933829943338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,65536,0.039775999387105306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,65536,0.036817065874735516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,2560,0.00772159993648529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,51200,0.03249280055363973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,51200,0.031828266382217404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,2048,0.006859733164310456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,16384,0.01346879998842875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,16384,0.020680532852808634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,16384,0.024873600403467814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,12288,0.011538133025169373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,1536,0.006548266609509785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,12288,0.019339734315872194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,10240,0.010497066378593444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,1024,0.005831466615200042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,768,0.005443199972311656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,8192,0.009412266810735067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,8192,0.01800853411356608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,256,0.0049002667268117275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,7168,0.008661333719889324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,7168,0.01835306684176127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,1024,128,0.004705066482226053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,10240,0.018653867642084758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,6144,0.008313600222269695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,6144,0.01781546672185262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,5120,0.007253333429495494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,5120,0.018039466937383015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,65536,0.08418986797332764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,4096,0.006422399977842967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,4096,0.017116800944010416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,51200,0.06730986436208089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,3584,0.006658133367697399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,3584,0.017101866006851197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,12288,0.019857066869735717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,3072,0.0061141331990559895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,3072,0.01660053332646688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,10240,0.017309866348902383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,2560,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,2560,0.01628266672293345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,8192,0.014818132917086283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,2048,0.0054506664474805195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,2048,0.015924266974131265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,7168,0.013075199723243714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,1536,0.004731733103593191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,1536,0.015622400244077048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,6144,0.011902933319409687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,1024,0.004007466634114583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,1024,0.015222400426864624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,5120,0.010622933506965637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,768,0.0036746665835380556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,768,0.014846932888031007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,4096,0.009320533275604248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,3584,0.008929066856702169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,512,0.0034527999659379324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,512,0.01490133305390676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,3072,0.008038400113582611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,256,0.002998399982849757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,256,0.01454080045223236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,2560,0.007658666869004567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,128,0.0028607999285062153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,128,0.01458346645037333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,128,0.004804266492525736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,64,0.002739199995994568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,64,0.014636799693107605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,768,32,0.0027445333699385325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,768,32,0.014731733004252115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,65536,0.028435200452804565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,65536,0.03020053307215373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,2048,0.006900266806284587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,51200,0.023520000775655112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,51200,0.026054400205612182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,51200,0.06770239671071371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,16384,0.010696533322334289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,16384,0.019203199942906698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,1536,0.006468266745408376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,12288,0.00922879974047343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,12288,0.017576533555984496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,1024,0.005689600110054016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,10240,0.008674133817354839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,10240,0.017753599087397258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,768,0.00544106662273407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,8192,0.007512533167997997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,8192,0.017672532796859743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,8192,0.014252799749374389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,7168,0.00676800012588501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,7168,0.017921066284179686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,6144,0.006369066735108693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,6144,0.01769386728604635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,12288,0.01983786622683207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,512,0.005236266553401947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,5120,0.006636799871921539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,5120,0.01803626616795858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,4096,0.0060362666845321655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,4096,0.017514665921529136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,768,256,0.004915200173854828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,4096,0.009346133470535279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,3584,0.006493866443634033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,3584,0.017242666085561117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,3584,0.00890239973862966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,3072,0.006146133442719777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,3072,0.016755199432373045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,2560,0.006090666850407918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,2560,0.016450132926305136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,65536,0.08429653644561767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,2048,0.005453866720199585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,2048,0.0162282665570577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,16384,0.024820266167322795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,1536,0.004856533308823904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,1536,0.015780267119407655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,10240,0.017118932803471883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,1024,0.004107733319203059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,1024,0.015236266454060874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,7168,0.013079466422398886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,768,0.0037216000258922578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,768,0.015001599987347921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,6144,0.011731200416882833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,512,0.0034453332424163817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,512,0.01493013302485148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,5120,0.010567466417948406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,256,0.003159466634194056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,256,0.01470186710357666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,256,0.004932266473770141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,128,0.0028522667785485585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,3072,0.008101333181063335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,128,0.014601600170135499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,64,0.002712533374627431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,2560,0.007620266576608021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,64,0.014564266800880432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,512,32,0.002775466690460841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,512,32,0.0144896000623703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,65536,0.01753173271814982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,65536,0.024884267648061117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,51200,0.013965866963068643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,65536,0.08331306775410971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,51200,0.02220906615257263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,16384,0.007901866734027863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,16384,0.01839146614074707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,2048,0.006841599941253662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,12288,0.006472533444563548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,12288,0.01794346570968628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,12288,0.0191210667292277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,10240,0.006331733365853627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,10240,0.01838399966557821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,1536,0.006551466882228851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,8192,0.006141866743564606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,8192,0.017798399925231932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,8192,0.014257066448529563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,1024,0.0057781333724657696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,7168,0.006082133452097575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,7168,0.017864533265431724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,6144,0.005942399799823761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,6144,0.01806400020917257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,768,0.005386666456858317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,512,0.005128533144791921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,5120,0.006318933268388112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,5120,0.018227199713389076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,512,128,0.0047423998514811196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,4096,0.005958400170008342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,4096,0.017552000284194947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,51200,0.0663541316986084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,3584,0.006400000055631001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,3584,0.016850133736928306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,16384,0.024043732881546022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,3072,0.006065066655476888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,3072,0.016839466492335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,10240,0.016619732975959776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,2560,0.005980800092220307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,2560,0.016390400131543477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,7168,0.013190399607022604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,2048,0.005406933526198069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,2048,0.01609386702378591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,6144,0.01179200013478597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,1536,0.0047199999292691554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,1536,0.015377066532770791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,5120,0.010488532980283101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,1024,0.003929600119590759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,1024,0.015109333395957946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,768,0.0036117332677046456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,768,0.015000533064206442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,768,0.005323733389377594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,512,0.0033461332321166994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,512,0.014738133549690247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,4096,0.009316266576449076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,256,0.0030410667260487873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,256,0.014774399995803832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,3584,0.008825600147247314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,128,0.0028277332584063213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,128,0.01452906628449758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,3072,0.008040533463160197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,64,0.002629333237806956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,64,0.014525866508483887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,256,32,0.0026176000634829206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,256,32,0.014553599556287131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,65536,0.011944533387819926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,65536,0.02145493427912394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,65536,0.08353493213653565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,51200,0.01030613382657369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,2560,0.007566933333873749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,51200,0.019292799631754558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,16384,0.006056533257166544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,16384,0.018407466014226277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,16384,0.02395413319269816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,12288,0.005902933577696482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,12288,0.018228266636530557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,2048,0.006870399912198384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,10240,0.0061152001221974695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,10240,0.017934934298197428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,1536,0.0064629331231117245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,8192,0.006007466713587443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,1024,0.0056981335083643595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,8192,0.017233065764109292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,7168,0.005919999877611796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,7168,0.018004266421000163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,512,0.005099733173847198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,6144,0.005774933099746704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,6144,0.017220266660054526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,256,0.0048320000370343525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,5120,0.006214400132497152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,5120,0.018012799819310508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,8192,0.014276267091433207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,256,128,0.004701866706212362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,4096,0.0057205334305763245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,4096,0.017004799842834473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,3584,0.0061749334136645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,3584,0.016886399189631144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,51200,0.06604373455047607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,3072,0.00589333325624466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,3072,0.016577066977818807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,3072,0.008105599880218506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,10240,0.016800000270207723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,2560,0.005750399827957153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,2560,0.016360533237457276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,2048,0.005243733525276184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,2048,0.015837867061297098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,7168,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,1536,0.004589866598447164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,1536,0.01530026694138845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,6144,0.011706667145093282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,1024,0.003942399968703588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,1024,0.015108266472816467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,12288,0.01908479928970337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,5120,0.010548266768455505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,768,0.0035797332723935447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,768,0.005400533477465311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,512,0.0033418667813142145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,512,0.01479039986928304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,4096,0.009244799613952637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,256,0.0029311999678611755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,256,0.014599466323852539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,3584,0.008762666583061218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,128,0.002834133307139079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,128,0.0144896000623703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,768,0.014843733112017313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,128,0.004946133494377137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,64,0.0027530667682488757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,64,0.014436266819636025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,128,32,0.002734933296839396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,128,32,0.014495999614397685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,65536,0.008967467149098714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,65536,0.01907306710879008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,51200,0.007458133498827617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,51200,0.019228800137837728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,16384,0.005963733295599619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,16384,0.017681066195170084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,12288,0.0059338668982187905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,12288,0.017309866348902383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,1536,0.006486399968465169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,10240,0.0061365331212679545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,8192,0.005915733178456625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,10240,0.017661867539087932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,8192,0.017453867197036742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,7168,0.005939200023810069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,7168,0.017780266205469766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,6144,0.005755733450253805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,5120,0.006075733403364817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,6144,0.017529600858688356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,4096,0.005645866692066193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,1024,0.005571199953556061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,4096,0.017479467391967773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,3584,0.006093866626421611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,3584,0.016693333784739174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,3072,0.005731200178464254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,2560,0.005643733342488607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,3072,0.016445866227149962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,2048,0.005115733544031779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,2560,0.01600320041179657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,1536,0.004421333471934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,2048,0.015590399503707886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,1024,0.003828266759713491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,1536,0.015404799580574035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,768,0.0035775999228159585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,1024,0.015236266454060874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,512,0.0032106667757034303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,768,0.014846932888031007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,2560,0.007542400062084198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,256,0.002963199963172277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,512,0.005064533154169718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,128,0.0027989332874615988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,512,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,256,0.014776532848676046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,64,0.0026101333399613695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,128,0.014501333236694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,64,32,0.0026133333643277483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,64,0.014481066664059957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,65536,0.008920533458391826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,32,0.014467199643452963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,51200,0.008060800035794576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,2048,0.006908800204594929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,65536,0.019873066743214925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,16384,0.005907199780146281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,51200,0.019820799430211387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,15,128,256,0.0047882666190465295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,10240,0.0060576001803080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,12288,0.00582826683918635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,16384,0.017869865894317626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,10240,0.01765226721763611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,12288,0.01744426687558492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,8192,0.0059690664211908976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,7168,0.005798399945100148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,8192,0.017505067586898803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,7168,0.01790613333384196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,5120,0.005939200023810069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,6144,0.01757226586341858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,3584,0.006053333481152853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,5120,0.017876267433166504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,4096,0.01737066706021627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,3072,0.005704533557097117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,3584,0.01679146687189738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,2560,0.0056970665852228795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,3072,0.016693333784739174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,2048,0.0050357331832249965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,1536,0.004461866617202759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,2560,0.01606186628341675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,1024,0.0037962667644023894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,2048,0.015761066476504007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,1536,0.015455999970436096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,1024,0.015239466230074564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,512,0.003171200056870779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,768,0.014973866939544677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,256,0.0029418667157491045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,512,0.014658133188883463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,128,0.0027253332237402597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,64,5120,0.017749333381652833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,256,0.01439573367436727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,64,0.0026762666801611584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,128,0.014380799730618796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,32,0.002603733291228612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,64,0.014486400286356607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,15,32,32,0.014342400431632995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,16384,0.39229545593261717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,6144,0.005637333293755849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,4096,0.005592533449331919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,16384,0.7473557154337566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,15,32,768,0.003472000112136205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,16384,0.3570293426513672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,12288,0.29830614725748694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,12288,0.5671103795369465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,51200,1.0969589233398438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,51200,1.183231989542643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,10240,0.25062506993611655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,10240,0.23991146087646484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,10240,0.49219627380371095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,7168,0.18074453671773275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,8192,0.2407072067260742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,7168,0.33038078943888344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,8192,0.40927785237630204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,51200,2.340772247314453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,6144,0.1593066692352295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,6144,0.28552961349487305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,5120,0.13408212661743163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,5120,0.28012905120849607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,4096,0.19360000292460125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,4096,0.126036270459493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,3584,0.19774932861328126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,3584,0.09906240304311117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,3072,0.14546666145324708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,3072,0.08752000331878662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,8192,0.1818559964497884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,2560,0.12464000384012859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,2560,0.075654403368632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,12288,0.2698783874511719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,7168,0.1598687966664632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,2048,0.09774293104807535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,6144,0.14792426427205402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,2048,0.06598613262176514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,2048,0.051130668322245276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,1536,0.08740586439768473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,1536,0.052052267392476405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,1024,0.05117119948069254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,1024,0.040004265308380124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,5120,0.12774399916330975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,1024,0.029029333591461183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,768,0.040090668201446536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,768,0.03439146677652995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,512,0.029572266340255737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,512,0.0278442660967509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,256,0.015658666690190635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,256,0.0211733341217041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,256,0.012262399991353352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,128,0.009287466605504353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,4096,0.09488960107167563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,128,0.01839146614074707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,64,0.006419200201829274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,128,0.010026666522026061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,64,0.016105600198109946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,65536,32,0.00461760014295578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,65536,32,0.016548267006874083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,3584,0.08420693079630534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,3072,0.07293866475423177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,2560,0.06224746704101562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,65536,1.2097216288248698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,1536,0.04016960064570109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,768,0.023164800802866616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,65536,512,0.01738133430480957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,65536,2.3607765197753907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,16384,0.3194346745808919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,16384,0.6127861022949219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,51200,1.851855977376302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,12288,0.24071253140767418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,51200,0.9559114456176758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,12288,0.5360469182332357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,10240,0.20384319623311362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,10240,0.4500981330871582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,8192,0.1661706606547038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,8192,0.35665388107299806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,7168,0.25821119944254556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,7168,0.1465354601542155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,6144,0.22273386319478355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,6144,0.12775786717732746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,65536,1.1576053619384765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,6144,0.11569386323293049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,5120,0.1091487963994344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,5120,0.21777920722961425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,16384,0.2961834589640299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,4096,0.15835307439168295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,4096,0.0913418690363566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,12288,0.22407892545064292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,3584,0.13129173119862875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,3584,0.08786453406016032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,3584,0.07060800393422445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,51200,0.9053792317708333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,3072,0.11529706319173176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,3072,0.07223146756490072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,2560,0.09816640218098958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,2560,0.062062935034434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,10240,0.21768320401509605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,2048,0.07828053633371988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,8192,0.1751306692759196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,1536,0.05940159956614176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,1536,0.04762453238169352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,1536,0.03410559892654419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,1024,0.040830934047698976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,7168,0.13324800332387288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,1024,0.03710826635360718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,2048,0.05813546578089396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,768,0.031641600529352824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,768,0.029419734080632524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,5120,0.09767680168151856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,512,0.025016532341639204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,4096,0.07933546702067057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,512,0.02560960054397583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,256,0.012414933244387309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,256,0.0195850670337677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,3072,0.06929813226064047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,2560,0.05221653381983439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,128,0.00844053328037262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,128,0.016033066312472026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,64,0.004268800218900045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,2048,0.04300053517023723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,64,0.016009599963823954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,51200,32,0.004253866771856943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,51200,32,0.015995732943216958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,1024,0.024913066625595094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,65536,0.3768522580464681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,65536,0.7425130844116211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,51200,0.2978613217671712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,51200,0.5896053314208984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,51200,0.28839146296183266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,16384,0.18367679913838703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,768,0.01995413303375244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,12288,0.08503146966298422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,12288,0.13943467140197754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,10240,0.1161845366160075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,10240,0.07269120216369629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,10240,0.06142186721165975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,512,0.016165332992871602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,8192,0.09421226978302003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,8192,0.06178346474965414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,256,0.010856533050537109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,7168,0.08266026973724365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,7168,0.056289064884185794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,6144,0.07206186453501383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,51200,128,0.00826453318198522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,6144,0.050619733333587644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,5120,0.06027733484903971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,16384,0.12368213335673015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,5120,0.04477013349533081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,4096,0.04915519952774048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,4096,0.038593065738677976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,65536,0.35653759638468424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,3584,0.05058240095774332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,3584,0.03576426506042481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,16384,0.11036159992218017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,3072,0.041094398498535155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,12288,0.07256853580474854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,3072,0.033030400673548385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,2560,0.03241173426310222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,2560,0.03009386658668518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,2560,0.020009599129358926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,2048,0.02636373241742452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,2048,0.028856533765792846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,8192,0.05019306739171346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,1536,0.020849066972732543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,1536,0.023490132888158162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,7168,0.04477333227793376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,1024,0.01463573376337687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,6144,0.039308798313140866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,1024,0.020679465929667153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,768,0.011577600240707397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,768,0.019350399573644005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,5120,0.038174935181935626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,512,0.009826133648554485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,512,0.017570134003957114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,4096,0.028140799204508467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,256,0.005721599857012431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,256,0.01530239979426066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,3584,0.02555519938468933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,128,0.003491200009981791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,128,0.015050666530927024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,3072,0.022541866699854533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,64,0.003268266717592875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,64,0.0151936004559199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,16384,32,0.003310933212439219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,16384,32,0.015060266852378846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,2048,0.01667520006497701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,1536,0.013748266299565635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,1024,0.010735999544461567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,65536,0.5550986607869466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,65536,0.2952661196390788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,768,0.009244799613952637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,16384,0.14494293530782063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,51200,0.23634986877441405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,51200,0.22120745976765951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,16384,0.08953706423441568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,51200,0.43659092585245773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,12288,0.11827627023061116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,12288,0.06852906545003255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,256,0.005314133564631144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,10240,0.09367573261260986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,10240,0.04877226750055949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,8192,0.07610133488972982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,8192,0.05033813317616781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,512,0.007843199868996937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,7168,0.06654719909032186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,16384,128,0.004935466746489207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,7168,0.04672319889068603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,6144,0.059284265836079916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,10240,0.059012266000111904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,6144,0.04169706503550212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,6144,0.03185813426971436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,5120,0.045765332380930585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,5120,0.03760106563568115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,4096,0.037774932384490964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,4096,0.03276159962018331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,65536,0.283676815032959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,3584,0.03759146531422933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,3584,0.030437332391738892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,16384,0.08765760262807211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,12288,0.0670805295308431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,3072,0.033030400673548385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,3072,0.02835200031598409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,2560,0.024630399545033772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,2560,0.02569173375765483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,8192,0.04053759972254435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,2048,0.020501333475112914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,2048,0.02358293334643046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,1536,0.016141866644223533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,1536,0.021601066986719767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,7168,0.0363103985786438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,1024,0.012291199962298075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,1024,0.01950719952583313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,5120,0.027273599306742353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,768,0.01004266639550527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,768,0.01829973260561625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,4096,0.023191465934117635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,512,0.007663999994595845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,512,0.01570453345775604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,512,0.005745066702365876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,256,0.004271999994913737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,256,0.015390933553377787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,3584,0.021409066518147786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,128,0.0035402665535608927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,128,0.014934399724006652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,3072,0.01890453298886617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,64,0.0032437334458033243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,2560,0.01677653392155965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,64,0.01490133305390676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,12288,32,0.0030591999491055804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,12288,32,0.014906666676203408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,2048,0.013860266407330832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,65536,0.2608725388844808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,1536,0.011787733435630799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,65536,0.48095467885335286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,51200,0.20348052978515624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,51200,0.3787477175394694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,16384,0.1255754629770915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,1024,0.009324799974759419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,16384,0.0862218697865804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,12288,0.09589440027872721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,12288,0.059374932448069254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,768,0.008260266482830047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,10240,0.07928319772084555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,10240,0.05378133455912272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,10240,0.042081065972646076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,8192,0.06747946739196778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,256,0.00513919989267985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,8192,0.04459626674652099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,7168,0.05328746636708578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,7168,0.04113386472066243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,6144,0.04679893255233765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,6144,0.03726079861323039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,65536,0.2404810587565104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,5120,0.04459199905395508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,5120,0.03365439971288045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,5120,0.024051199356714882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,12288,128,0.004962133367856344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,4096,0.03191039959589641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,4096,0.02974826693534851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,3584,0.028516266743342084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,3584,0.028359466791152955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,12288,0.04933653275171916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,3072,0.024972800413767496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,3072,0.026009599367777508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,8192,0.03460906744003296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,2560,0.022011733055114745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,2560,0.024267733097076416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,7168,0.031274666388829545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,2048,0.017476266622543334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,2048,0.022375466426213582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,51200,0.18939305941263834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,16384,0.06425386667251587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,1536,0.015689599514007568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,1536,0.020273067553838096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,1024,0.010228266318639118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,1024,0.018424532810846963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,6144,0.027754666407903035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,768,0.008408533533414204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,768,0.01672533353169759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,4096,0.020853332678476968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,512,0.006674133241176605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,512,0.015602133671442666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,512,0.0056309332450230915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,3072,0.01663146714369456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,256,0.003688533355792364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,3584,0.018718934059143065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,256,0.015050666530927024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,128,0.0033141332368055976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,128,0.014784000317255654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,2560,0.014773333072662353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,64,0.0030410667260487873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,64,0.01469546655813853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,10240,32,0.0030389333764712016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,2048,0.012662399808565775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,10240,32,0.014781866470972696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,65536,0.20148159662882487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,65536,0.3758453369140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,1536,0.01088853379090627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,51200,0.296830940246582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,1024,0.008913066983222962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,51200,0.1609610716501872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,16384,0.06097813447316488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,16384,0.09273066520690917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,12288,0.07079040209452311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,12288,0.05035519997278849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,768,0.007063466807206471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,12288,0.04029546578725179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,10240,0.044392534097035724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,10240,0.03429866631825765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,8192,0.04808106819788615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,8192,0.03826773166656494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,7168,0.04254613320032756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,7168,0.03521066506703695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,7168,0.02598080039024353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,6144,0.037514666716257736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,256,0.005127466718355815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,6144,0.03269226749738057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,5120,0.0316650668780009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,10240,128,0.004964266717433929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,5120,0.020056533813476562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,5120,0.02964693307876587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,16384,0.05144213438034058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,4096,0.025795199473698932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,4096,0.0264682670434316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,3584,0.0232149342695872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,3584,0.015826132893562318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,3584,0.025294933716456098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,3072,0.02036799987157186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,10240,0.059349334239959715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,3072,0.023596799373626708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,2560,0.017548799514770508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,2560,0.021950932343800862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,2048,0.01441493332386017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,2048,0.020549333095550536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,65536,0.18468480110168456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,1536,0.01325653294722239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,1536,0.01917546590169271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,51200,0.15118826230367025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,1024,0.009781333804130554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,1024,0.017451733350753784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,8192,0.028776532411575316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,768,0.007281066477298736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,768,0.015124266346295675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,6144,0.02293333411216736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,512,0.005748266478379568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,512,0.015505066514015198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,512,0.005612800021966299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,256,0.0037439999481042228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,256,0.014910933375358582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,4096,0.01709653337796529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,128,0.003252266595760981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,3072,0.013730133573214212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,128,0.014603733023007711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,64,0.0030954666435718536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,2560,0.012626133362452188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,64,0.014636799693107605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,8192,32,0.003017599880695343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,8192,32,0.014686933159828186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,2048,0.011100799838701884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,65536,0.185260804494222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,65536,0.16423786481221517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,65536,0.32643947601318357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,51200,0.1304458697636922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,51200,0.14950720469156903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,51200,0.25816960334777833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,1536,0.009846400221188862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,16384,0.04638719956080119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,16384,0.05776426792144775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,16384,0.09252693653106689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,12288,0.0643776019414266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,12288,0.046044798692067464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,10240,0.05557760000228882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,10240,0.041495466232299806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,8192,0.04551573197046916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,8192,0.03602026700973511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,7168,0.03887786865234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,1024,0.008050133287906647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,7168,0.033395198980967204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,6144,0.03428266843159993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,6144,0.030671999851862593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,768,0.005921066800753275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,5120,0.02850666642189026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,5120,0.028150399525960285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,256,0.0051146666208903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,8192,128,0.004931200047334035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,4096,0.02344640096028646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,4096,0.02537920077641805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,12288,0.03631680011749268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,3584,0.020989867051442464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,10240,0.031439999739329025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,3584,0.023949867486953734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,3072,0.018557866414388023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,3072,0.02291946609814962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,7168,0.02351893385251363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,2560,0.01590293347835541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,2560,0.021309866507848104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,6144,0.02098133365313212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,2560,0.011723732948303223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,2048,0.013150933384895324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,2048,0.01034773290157318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,2048,0.020087466637293497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,1536,0.010637866457303365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,1536,0.01871253252029419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,1024,0.008180266618728638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,1024,0.01588053305943807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,8192,0.02606826623280843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,768,0.006788266698519389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,5120,0.018119466304779053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,768,0.015406933426856995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,512,0.004099199920892716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,512,0.015352533260981242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,512,0.005492266515890757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,256,0.003389866650104523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,256,0.014765866597493491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,128,0.0030901332696278887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,4096,0.015528532862663268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,128,0.014807466665903726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,64,0.002898133297761281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,64,0.014747732877731323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,7168,32,0.002885333448648453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,7168,32,0.014667733510335287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,65536,0.28196481068929036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,65536,0.1593621253967285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,3584,0.014342400431632995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,256,0.00522986650466919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,3072,0.012770133217175803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,65536,0.14456213315327962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,51200,0.2229973316192627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,16384,0.07589653333028158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,51200,0.1280234654744466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,16384,0.05115093390146891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,12288,0.058317867914835606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,12288,0.04248746633529663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,10240,0.04840213457743327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,1536,0.009121066331863404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,10240,0.03756800095240275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,8192,0.038116268316904706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,8192,0.033802668253580734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,1024,0.006760533154010773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,7168,0.03470933437347412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,7168,0.03181866606076558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,7168,0.021156267325083414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,6144,0.030300800005594892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,6144,0.029725867509841918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,6144,0.018915200233459474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,768,0.005971199770768484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,5120,0.025678932666778564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,5120,0.02725013295809428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,4096,0.021556266148885093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,4096,0.024514132738113405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,4096,0.01399893363316854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,3584,0.01938986579577128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,7168,128,0.0049792001644770306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,12288,0.032585599025090534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,3584,0.023373866081237794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,3072,0.017038933436075845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,3072,0.022422399123509726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,2560,0.014663466811180114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,2560,0.020770132541656494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,51200,0.11459840138753255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,16384,0.041127467155456544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,2048,0.013699199755986533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,2048,0.019441066185633342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,1536,0.010011733571688334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,1536,0.018441599607467652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,10240,0.027876265843709308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,1536,0.008803199728329976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,1024,0.007551999886830647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,1024,0.005798399945100148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,1024,0.015783466895421348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,768,0.006401066482067108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,768,0.015380266308784484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,768,0.0055861334005991616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,512,0.00391146664818128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,8192,0.02328853408495585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,512,0.014966400464375815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,256,0.003349333256483078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,256,0.00517546683549881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,256,0.01474240024884542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,128,0.0031210665901501974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,128,0.014667733510335287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,128,0.004816000163555145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,64,0.0028277332584063213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,64,0.014681599537531533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,6144,32,0.0028970666229724885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,6144,32,0.014615466197331747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,65536,0.24158080418904623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,65536,0.13985493977864583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,3584,0.013148799538612366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,2560,0.010988799730936687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,65536,0.12470826307932537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,51200,0.1914090633392334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,512,0.005342933535575867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,51200,0.10862507025400799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,16384,0.07088320255279541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,12288,0.049253332614898684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,12288,0.037758934497833255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,12288,0.02886506716410319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,10240,0.04168320099512736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,10240,0.034069331487019856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,2048,0.009689600268999735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,5120,0.01660160024960836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,8192,0.03380586703618367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,8192,0.030075732866923017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,8192,0.02063039938608805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,7168,0.028242133061091107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,7168,0.028267733256022137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,16384,0.04605226516723633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,6144,0.02503040035565694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,6144,0.026393600304921466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,5120,0.021500800053278604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,6144,3072,0.011858133474985759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,5120,0.024425599972407022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,4096,0.017745065689086913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,4096,0.022447999318440756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,3584,0.015972266594568886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,3584,0.02130133310953776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,51200,0.0996298631032308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,3072,0.01402346690495809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,16384,0.03605226675669353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,3072,0.020691200097401937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,2560,0.012228266398111979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,2560,0.01982933282852173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,10240,0.024675200382868447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,2048,0.010429867108662923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,2048,0.01876693367958069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,7168,0.019050665696461997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,6144,0.01691306630770365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,1536,0.008638933300971985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,1536,0.016722132762273155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,1024,0.006849066913127899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,1024,0.015064533551534018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,5120,0.014737066626548768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,768,0.00547626664241155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,4096,0.012793599565823873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,768,0.015243732929229736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,512,0.0036864000062147772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,512,0.015056000153223673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,512,0.005474133292833964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,256,0.003268266717592875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,3584,0.012049067020416259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,256,0.014909866452217101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,128,0.002998399982849757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,128,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,128,0.004863999783992767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,64,0.0028917332490285236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,64,0.014660267035166421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,5120,32,0.0029557332396507262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,5120,32,0.014626133441925048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,65536,0.18571306864420573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,3072,0.010950400431950887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,65536,0.11107733249664306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,2560,0.010416000088055929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,51200,0.147214937210083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,51200,0.09003946781158448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,16384,0.050230399767557776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,16384,0.039691734313964847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,2048,0.008993066350618998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,12288,0.039707732200622556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,12288,0.03396053314208984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,1536,0.007866666714350382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,10240,0.03350186745325724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,65536,0.10729706287384033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,10240,0.030792532364527385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,1024,0.006183466811974844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,8192,0.02794133424758911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,8192,0.02733866572380066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,768,0.005692799886067709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,7168,0.024954666694005333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,7168,0.02584213415781657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,6144,0.02226240038871765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,6144,0.024267733097076416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,5120,256,0.005075199902057648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,5120,0.019452800353368126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,5120,0.02259946664174398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,8192,0.018377600113550822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,4096,0.016064000129699708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,4096,0.02093013326327006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,16384,0.03146666685740153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,51200,0.0865109364191691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,3584,0.014434132973353067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,3584,0.020897066593170165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,3072,0.011993599931399028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,12288,0.02524799903233846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,3072,0.019723733266194664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,10240,0.021964800357818604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,2560,0.010493866602579753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,2560,0.018797866503397622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,7168,0.01689280072848002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,2048,0.0090421328941981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,2048,0.017832533518473307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,6144,0.015102932850519816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,1536,0.00766186664501826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,1536,0.01583999991416931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,1536,0.006931200126806895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,1024,0.005789866546789805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,5120,0.01333440045515696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,1024,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,768,0.003902933249870936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,768,0.015191466609636942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,4096,0.011729066570599873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,512,0.003553066651026408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,512,0.015041066209475198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,3584,0.011108266313870747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,256,0.003268266717592875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,256,0.014622933665911355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,256,0.005037866532802582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,128,0.003065599997838338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,128,0.014510933558146158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,3072,0.010207999746004741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,64,0.0028053333361943563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,64,0.01477226714293162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,4096,32,0.002914133419593175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,4096,32,0.014797866344451904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,65536,0.17014613151550292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,65536,0.10337599913279216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,51200,0.1343722661336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,2560,0.009716266393661499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,51200,0.08268799781799316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,2048,0.008433066805203756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,16384,0.048485334714253744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,16384,0.03702720006306966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,12288,0.03493119875590007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,12288,0.03197759985923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,1024,0.005749333401521047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,10240,0.03094826738039653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,768,0.005593599875768026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,10240,0.02900586724281311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,10240,0.021042132377624513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,8192,0.025579732656478883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,8192,0.026143999894460042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,7168,0.022729599475860597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,7168,0.025143466393152875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,51200,0.08528746763865153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,6144,0.02030506730079651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,6144,0.023110399643580117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,512,0.005287466446558634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,5120,0.01748266617457072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,5120,0.021953066190083824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,4096,128,0.004791466891765595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,4096,0.014755200346310934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,4096,0.0206986665725708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,4096,0.011546666423479717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,3584,0.01317759950955709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,3584,0.02020373344421387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,65536,0.10384106636047363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,3072,0.012176000078519185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,3072,0.019146666924158732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,16384,0.030204800764719646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,2560,0.00978773335615794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,2560,0.01864746610323588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,2560,0.009550933043162029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,2048,0.008411733309427898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,12288,0.02418346603711446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,2048,0.016952532529830932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,2048,0.007747200131416321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,1536,0.00720000018676122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,1536,0.01553600033124288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,1024,0.004826666911443075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,1024,0.015261866648991904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,1024,0.00614933321873347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,768,0.003882666677236557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,8192,0.017110399405161538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,768,0.015166933337847391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,512,0.003555200000603994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,512,0.014934399724006652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,7168,0.016174933314323424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,256,0.0032490665713946023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,256,0.014703999956448874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,6144,0.0146506667137146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,128,0.0029919999341169994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,5120,0.01300266683101654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,128,0.014531200130780539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,128,0.004917333523432413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,64,0.0028245332340399425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3584,32,0.0029088000456492106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,32,0.014577066898345948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,65536,0.14557120005289714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,65536,0.09043947060902914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,3584,0.011061333616574605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,51200,0.11480000019073486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,65536,0.0955519994099935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,51200,0.07305813630421956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,3072,0.010057600339253743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,16384,0.04053013324737549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,16384,0.034858667850494386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,16384,0.02808213432629903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3584,64,0.01458026667435964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,12288,0.03204480012257894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,12288,0.030133332808812457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,10240,0.02729066610336304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,1536,0.006728533407052357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,10240,0.027965867519378663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,8192,0.023264000813166298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,8192,0.025218133131663007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,8192,0.016723199685414632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,7168,0.02097066640853882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,7168,0.024016000827153525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,768,0.0055861334005991616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,7168,0.015452800194422403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,6144,0.018313600619633993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,512,0.005351466437180838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,6144,0.022692267100016275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,5120,0.016029866536458333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,5120,0.02097599903742472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,4096,0.01395626664161682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,4096,0.019901865720748903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,4096,0.011004799604415893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,3584,0.012749866644541422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,3584,0.01946880022684733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3584,256,0.005102933446566264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,3072,0.011416533589363098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,3072,0.018760534127553304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,3072,0.009526399771372478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,2560,0.00906773308912913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,51200,0.07620906829833984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,2560,0.017892267306645712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,2048,0.007890133559703827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,2048,0.016250666975975037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,12288,0.022530132532119752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,1536,0.006828799843788147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,1536,0.01572160025437673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,10240,0.01956160068511963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,1024,0.0044608001907666525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,1024,0.0060032000144322716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,1024,0.015770666797955833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,768,0.003907199949026108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,768,0.01516480048497518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,512,0.0035071998834609987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,512,0.015060266852378846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,6144,0.013479466239611307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,256,0.003201066702604294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,5120,0.01252906620502472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,256,0.014959999918937683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,128,0.002958933264017105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,128,0.01439786652723948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,3584,0.010644267002741497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,64,0.002775466690460841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,64,0.014654933412869772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,3072,32,0.0028245332340399425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,3072,32,0.014653866489728292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,65536,0.12127466996510823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,65536,0.07832746505737305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,2560,0.008855467041333515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,2048,0.007344000041484833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,51200,0.0967146635055542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,51200,0.06371200084686279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,16384,0.033846398194630936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,16384,0.03157013257344564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,1536,0.006691200037797292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,12288,0.026630399624506633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,12288,0.02701866626739502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,768,0.005527466535568237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,10240,0.023139200607935586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,10240,0.02468693256378174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,512,0.005285333096981049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,8192,0.019319466749827065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,8192,0.023226666450500488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,256,0.0050453335046768185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,7168,0.017384533087412515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,3072,128,0.004841599861780802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,7168,0.02241599957148234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,6144,0.015501866738001505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,6144,0.021364265680313112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,6144,0.013689600427945457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,5120,0.013729066650072733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,5120,0.020474666357040407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,5120,0.012430933117866517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,4096,0.011769599715868632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,4096,0.019341866175333657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,65536,0.09372479915618896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,3584,0.010980266332626342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,3584,0.018743467330932618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,7168,0.01423679987589518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,3072,0.01011199951171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,3072,0.017432532707850137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,3072,0.008870399991671244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,2560,0.008307200173536937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,2560,0.01611840029557546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,51200,0.0745247999827067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,16384,0.027484800418217974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,2048,0.0072170664866765336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,2048,0.016177067160606386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,12288,0.02179946700731913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,1536,0.006135466694831848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,1536,0.015709867080052696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,1536,0.006555733581384023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,10240,0.019206400712331137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,1024,0.004301866888999939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,1024,0.015212800105412802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,8192,0.016452266772588094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,768,0.003884800026814143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,768,0.0055285334587097164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,768,0.015177599589029946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,512,0.003458133339881897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,512,0.014957867066065469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,256,0.003189333279927572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,256,0.014723199605941772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,256,0.005025066435337067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,128,0.0029440000653266907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,128,0.014475733041763306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,128,0.004882133503754934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,64,0.0027488000690937043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,64,0.014567466576894126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2560,32,0.00277866671482722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2560,32,0.014625066518783569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,4096,0.011002666751543681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,65536,0.09659732977549235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,65536,0.06416320006052653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,51200,0.07588160037994385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,3584,0.010182399551073711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,51200,0.05429866711298624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,16384,0.027812266349792482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,51200,0.06856640179951987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,16384,0.028011733293533327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,12288,0.022124799092610677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,12288,0.024272000789642333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,2560,0.007933866480986278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,10240,0.019222400585810342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,10240,0.022949333985646567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,2048,0.007241599758466085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,8192,0.016075733304023742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,8192,0.021631999810536703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,1024,0.006110933423042297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,7168,0.01458453337351481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,7168,0.02091839909553528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,7168,0.014393599828084311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,6144,0.012997333208719888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,6144,0.012883200248082479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2560,512,0.005286400020122528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,6144,0.02050559918085734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,5120,0.011800533533096314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,4096,0.010276266932487487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,5120,0.01949013272921244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,4096,0.010428800185521444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,4096,0.01867199937502543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,3584,0.009672533472379048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,3584,0.009637332955996196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,3072,0.008941866954167684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,3072,0.01692906618118286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,65536,0.08538560072580972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,2560,0.007602133353551229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,2560,0.016246400276819863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,2560,0.007743999858697255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,2048,0.006727466483910878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,2048,0.01623466710249583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,16384,0.025437867641448973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,1536,0.004961066444714864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,1536,0.015650133291880287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,3584,0.017172267039616905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,1024,0.004093866546948751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,1024,0.015124266346295675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,1024,0.005757866799831391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,768,0.003737599899371465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,768,0.015118933717409768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,10240,0.01773973306020101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,512,0.00347626656293869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,512,0.014798933267593383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,512,0.005261866748332978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,256,0.003068800022204717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,8192,0.015530666708946228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,256,0.0145087997118632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,128,0.0028319999575614927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,256,0.004954666892687479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,128,0.014472533265749613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,12288,0.020472532510757445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,64,0.002678400029738744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,2048,32,0.002733866622050603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,64,0.014621866742769876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,2048,32,0.014333867033322654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,5120,0.011825066804885865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,65536,0.05373973449071249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,51200,0.05970026652018229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,51200,0.04522879918416341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,3072,0.008516266942024231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,16384,0.022485333681106567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,16384,0.024452267090479533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,2048,0.0070165331164995836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,12288,0.018173867464065553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,12288,0.023194666703542074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,65536,0.07500373522440593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,10240,0.015754666924476624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,1536,0.0065653334061304735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,65536,0.08445973396301269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,10240,0.02158400019009908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,8192,0.013366400202115377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,8192,0.020744532346725464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,8192,0.015096533298492431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,7168,0.012337066729863485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,7168,0.020092799266179403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,768,0.0054954667886098225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,6144,0.010959999759991963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,6144,0.01925546725591024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,2048,128,0.004698666433493296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,5120,0.010326400399208069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,5120,0.019332265853881835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,51200,0.06728106339772542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,4096,0.00915839970111847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,16384,0.025037866830825806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,4096,0.017374932765960693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,3584,0.008597333232561748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,3584,0.01662826637427012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,3584,0.008889599641164144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,3072,0.008161066472530365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,3072,0.016795732577641807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,3072,0.008193066716194153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,2560,0.0069919998447100324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,2560,0.01616426706314087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,12288,0.01998293399810791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,2048,0.005474133292833964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,2048,0.01611199975013733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,10240,0.01770240068435669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,1536,0.004825599988301595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,1536,0.015719466408093772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,1024,0.004030933231115341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,7168,0.014086400469144186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,4096,0.009798399607340495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,768,0.0037589333951473236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,768,0.015072000026702882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,6144,0.012708266576131185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,512,0.0034474665919939675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,512,0.014691199858983359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,5120,0.011354666948318482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,256,0.00311253344019254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,1024,0.015156267086664834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,256,0.0146506667137146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,128,0.0029002666473388673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,128,0.014514133334159851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,2560,0.007667199770609538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,64,0.0028533334533373516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,64,0.01439466675122579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1536,32,0.0027488000690937043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,2048,0.006829866766929626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,65536,0.05058026711146037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,65536,0.04161066611607869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,1536,0.006557866434256236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,51200,0.04061866601308187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,1024,0.00574186642964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,51200,0.03646506468454997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,16384,0.01602453291416168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,16384,0.02216106653213501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,16384,0.024881066878636678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,12288,0.013432533542315165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,768,0.005526400109132131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,12288,0.020091732343037925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,12288,0.019963733355204263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,10240,0.012114133437474568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,512,0.005235200126965841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,10240,0.01758293310801188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,10240,0.019629865884780884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,8192,0.010526933272679647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,7168,0.009914666414260864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,8192,0.019142399231592812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,7168,0.018355200688044228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,6144,0.008803199728329976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,6144,0.017768534024556477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,6144,0.012335999806722005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1536,32,0.0143477330605189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,5120,0.008571733037630718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,5120,0.010587732990582783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,5120,0.018126932779947917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,4096,0.008106666803359985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,4096,0.01742080052693685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,3584,0.007212799787521362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,3584,0.017001599073410034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,3584,0.00902186632156372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,3072,0.006487466891606649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,3072,0.016634666919708253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,256,0.00484799991051356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,2560,0.006206933160622915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,2560,0.016438399751981102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1536,128,0.004726399978001913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,2048,0.005480533341566721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,2048,0.016029866536458333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,7168,0.013351466258366904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,1536,0.004728533327579498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,1536,0.015631999572118124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,1536,0.006504533191521962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,1024,0.0040949332217375435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,1024,0.015222400426864624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,1024,0.005842133363087972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,768,0.003667200108369192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,768,0.015072000026702882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,65536,0.08358933130900065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,51200,0.06667093435923258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,512,0.003419733295838038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,512,0.014641066392262777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,8192,0.015017599860827128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,256,0.003101866692304611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,256,0.01458560029665629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,256,0.004955733319123586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,128,0.0028223998844623564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,128,0.014390400052070618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,128,0.004699733356634776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,64,0.0027146667242050173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,4096,0.009330133597056072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,64,0.014333867033322654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,1024,32,0.0028021333118279776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,1024,32,0.01432213286558787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,65536,0.039795200030008956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,65536,0.036414933204650876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,51200,0.03237546682357788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,51200,0.03159466584523519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,3072,0.008054399987061818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,16384,0.013512532909711203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,16384,0.02054506738980611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,16384,0.02492693265279134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,2560,0.007686399916807811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,12288,0.011522133151690166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,12288,0.019629865884780884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,10240,0.010614400108655293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,10240,0.018609066804250084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,2048,0.006955733398596446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,8192,0.009691733121871948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,8192,0.017492266496022542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,7168,0.008684800068537394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,768,0.00544213354587555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,7168,0.017756799856821694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,6144,0.008208000163237254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,6144,0.017241599162419637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,1024,512,0.0052255998055140175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,5120,0.007175466914971669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,5120,0.017964800198872886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,65536,0.08357333342234294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,4096,0.006380799909432728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,4096,0.017691733439763387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,51200,0.06722026666005453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,3584,0.006698666512966156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,3584,0.017733333508173625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,12288,0.019869865973790486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,3072,0.0062826668222745255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,3072,0.016458666324615477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,10240,0.01742080052693685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,2560,0.006056533257166544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,2560,0.016639999548594155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,8192,0.01472106675306956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,2048,0.005457066496213277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,2048,0.01602133313814799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,7168,0.013079466422398886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,1536,0.004721066852410635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,1536,0.015465600291887918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,6144,0.011852799852689107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,1024,0.004007466634114583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,1024,0.015290666619936624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,5120,0.01050879955291748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,768,0.003740799923737844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,768,0.015153066317240397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,768,0.0053962667783101406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,512,0.0033546666304270422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,512,0.01478506624698639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,4096,0.009327999750773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,256,0.003053866575161616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,256,0.014587733149528503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,3584,0.008983467022577922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,128,0.002882133424282074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,128,0.014537599682807923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,3072,0.008197333415349324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,64,0.0026922665536403658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,64,0.014402133226394654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,768,32,0.002682666728893916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,2560,0.007645866771539052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,768,32,0.014578133821487427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,65536,0.028332799673080444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,65536,0.029501867294311524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,51200,0.023605332771937052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,51200,0.026345600684483845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,2048,0.006931200126806895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,16384,0.01060693363348643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,16384,0.019186133146286012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,1536,0.006492800017197926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,12288,0.009221333265304565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,12288,0.017308799425760905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,1024,0.005719466507434845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,10240,0.008573866883913676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,10240,0.017500799894332886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,10240,0.017107200622558594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,8192,0.007492266595363617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,8192,0.017571200927098594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,512,0.0051242664456367494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,7168,0.006749866902828217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,7168,0.018132267395655315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,256,0.004885333279768625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,6144,0.006341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,768,128,0.004820266862710317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,6144,0.017661867539087932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,65536,0.08414080142974853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,5120,0.006385066608587901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,5120,0.018415999412536622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,51200,0.06702400048573812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,4096,0.006044800082842508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,4096,0.017476266622543334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,16384,0.024522666136423746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,3584,0.0064640000462532045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,3584,0.017214934031168617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,12288,0.019618133703867592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,3072,0.00613013356924057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,3072,0.016544000307718913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,8192,0.014473600188891092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,2560,0.006156800190607706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,2560,0.016456533471743265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,2560,0.007667199770609538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,2048,0.005493333439032236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,7168,0.012942933042844138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,2048,0.016254933675130208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,2048,0.006841599941253662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,1536,0.015546666582425437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,6144,0.011833600203196208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,1024,0.004127999891837438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,1024,0.01525973379611969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,768,0.0038293334345022834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,768,0.015075199802716575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,1536,0.0047541335225105286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,5120,0.010427733262379963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,512,0.0034272000193595886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,512,0.014844800035158793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,256,0.0030762667457262674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,256,0.014626133441925048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,256,0.004796800017356872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,128,0.0028927999238173166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,4096,0.009258666634559631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,128,0.014590932925542196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,64,0.002701866626739502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,64,0.014548266927401224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,512,32,0.0027285332481066385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,512,32,0.01446399986743927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,65536,0.017509333292643228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,3584,0.008899199962615966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,65536,0.023911466201146446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,51200,0.01394773324330648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,3072,0.008081066608428954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,51200,0.022380799055099487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,16384,0.00786133309205373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,51200,0.06608533461888631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,16384,0.018066134055455527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,16384,0.023846399784088135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,12288,0.017383466164271034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,1536,0.006493866443634033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,10240,0.006490666667620342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,10240,0.017942400773366292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,12288,0.00647680014371872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,8192,0.006132266422112783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,8192,0.018011732896169027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,8192,0.014199466506640116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,7168,0.006077866752942403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,7168,0.018258132537206016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,7168,0.012919466694196066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,1024,0.0056874667604764305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,6144,0.005995733539263407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,6144,0.017594667275746663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,768,0.005425066749254862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,5120,0.0063701331615448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,5120,0.017966934045155845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,4096,0.005981866518656413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,512,0.005144533514976501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,4096,0.017349332571029663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,4096,0.009231999516487122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,3584,0.006397866706053417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,3584,0.017195733388264973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,3072,0.006033066908518473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,3072,0.01689493258794149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,512,128,0.004866133133570353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,2560,0.0059914668401082356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,2560,0.016173866391181946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,65536,0.08392213185628256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,2048,0.005338666836420695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,2048,0.0160480002562205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,12288,0.019002666076024376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,1536,0.004605866471926371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,1536,0.015411200126012168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,10240,0.016593066851298015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,1024,0.004038399954636892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,1024,0.015074132879575094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,6144,0.011693867047627766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,5120,0.011136000355084736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,768,0.0036277333895365395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,768,0.014918399850527444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,512,0.0034229333202044168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,512,0.014808533589045205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,3584,0.008871466914812724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,256,0.002974933385848999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,3072,0.008020266890525818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,256,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,128,0.0027797333896160126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,2560,0.007578666508197785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,128,0.014493866761525472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,2048,0.006760533154010773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,64,0.0026421333352724713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,64,0.014459733168284097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,256,32,0.0026528000831604003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,1536,0.006428800026575724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,65536,0.01183573305606842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,256,32,0.014531200130780539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,65536,0.021258666117986044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,65536,0.08370453516642253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,51200,0.010247466961542766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,51200,0.01922453244527181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,16384,0.006038400034109751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,16384,0.017951999107996622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,16384,0.023882667223612465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,12288,0.005892266829808554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,12288,0.018029866615931193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,12288,0.01906986633936564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,10240,0.006084266801675161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,1024,0.005715199808279673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,10240,0.017694934209187826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,8192,0.005976533393065134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,8192,0.017459199825922648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,768,0.005371733506520589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,7168,0.00589333325624466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,7168,0.017809067169825235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,7168,0.012983466188112894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,6144,0.005793066819508871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,6144,0.01763413349787394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,256,0.004851200183232625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,5120,0.006172800064086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,5120,0.017797333002090455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,128,0.004693333307902018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,4096,0.005764266848564148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,4096,0.01707093318303426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,3584,0.0062389334042867025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,3584,0.01688213348388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,3584,0.008854400118192036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,3072,0.005914666752020518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,3072,0.016586666305859886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,3072,0.008082133531570435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,2560,0.005766400198141734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,256,512,0.005179733534653981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,51200,0.06574506759643554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,2560,0.016312533617019655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,2048,0.005082666873931885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,2048,0.015804800391197204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,2048,0.006742399930953979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,1536,0.0045962666471799215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,1536,0.015388799707094827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,1024,0.0038751999537150065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,1024,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,1024,0.005639466643333435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,768,0.0036469332873821257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,10240,0.01663146714369456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,768,0.014717866977055868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,512,0.0033269333342711128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,512,0.014798933267593383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,8192,0.014192000031471252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,256,0.003050666550795237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,256,0.014615466197331747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,256,0.004745600124200186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,128,0.0028010666370391845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,128,0.014405333002408347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,6144,0.011678933103879293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,64,0.0027082666754722597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,64,0.014428800344467163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,128,32,0.002699733277161916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,5120,0.010505599776903789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,128,32,0.014375467101732889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,65536,0.008565333485603333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,51200,0.007485866546630859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,65536,0.01967359979947408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,16384,0.005996799965699514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,51200,0.019309866428375243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,12288,0.0059562668204307554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,16384,0.017826133966445924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,10240,0.006064000229040781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,12288,0.017913599809010826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,4096,0.009154133001963298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,8192,0.005975466469923655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,7168,0.005786666770776113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,8192,0.017449599504470826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,6144,0.005755733450253805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,7168,0.017910399039586387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,5120,0.006010666489601135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,6144,0.01726400057474772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,4096,0.005700266857941946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,5120,0.017925333976745606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,3584,0.0060576001803080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,4096,0.01690773367881775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,3072,0.005862399935722351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,3584,0.016923733552296958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,2560,0.005719466507434845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,3072,0.016288000345230102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,2560,0.01650773286819458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,2048,0.005053866902987162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,2560,0.007606400052706401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,1536,0.004433066646258036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,2048,0.015794133146603904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,1024,0.0038783999780813852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,1536,0.015367466211318969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,768,0.0035562666753927866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,1024,0.0151829332113266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,512,0.0032885332902272543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,768,0.014888532956441245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,1536,0.006409599880377452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,256,0.0029834667841593427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,512,0.014797866344451904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,128,0.0027935999135176343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,256,0.014769066373507181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,64,0.002712533374627431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,128,0.014519466956456503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,64,32,0.0026389333109060925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,64,0.014393599828084311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,65536,0.009011200070381165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,32,0.014518400033315023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,768,0.005387733379999796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,51200,0.008090666433175405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,51200,0.01924053430557251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,16384,0.005880533158779145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,16384,0.017595734198888144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,12288,0.005805866420269012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,512,0.005109333495299021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,12288,0.017122133572896322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,8192,0.005927466849486033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,7168,0.0056981335083643595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,8192,0.017378133535385133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,6144,0.005605333546797434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,64,10240,0.017768534024556477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,7168,0.017620267470677693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,5120,0.005962666869163513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,6144,0.017451733350753784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,5120,0.017875200510025023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,4096,0.0055637334783871974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,3584,0.005972266693909963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,4096,0.01707093318303426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,3072,0.005727999905745188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,3584,0.01685439944267273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,2560,0.005660800139109293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,2048,0.00499839981396993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,2560,0.01623679995536804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,1536,0.00446720023949941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,2048,0.015899733702341715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,14,128,128,0.004679466784000397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,1024,0.0037791999677817024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,1536,0.015531733632087708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,768,0.00348693331082662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,1024,0.014979199568430582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,512,0.003218133250872294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,768,0.01493013302485148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,256,0.00296426663796107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,512,0.01461120049158732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,128,0.002794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,256,0.014477866888046264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,64,0.0025920001169045764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,128,0.014257066448529563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,32,0.002587733417749405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,64,0.014265599846839904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,32,0.014338133732477823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,65536,0.020060799519220986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,14,32,10240,0.006040533383687338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,10240,0.017887999614079796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,14,32,3072,0.016481066743532814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,16384,0.3914538701375326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,16384,0.7476437250773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,12288,0.5664693196614583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,12288,0.2980832099914551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,51200,1.1815914154052733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,10240,0.25015467007954917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,10240,0.4771114667256673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,8192,0.20483199755350748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,8192,0.4447157224019368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,51200,2.339906056722005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,7168,0.3303936004638672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,7168,0.18212693532307941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,6144,0.1611519972483317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,6144,0.28538452784220375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,5120,0.23759147326151528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,5120,0.13374293645222982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,16384,0.3575146675109863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,5120,0.11649386882781983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,4096,0.10976320107777912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,4096,0.1927669366200765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,3584,0.1684757391611735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,12288,0.2792127927144369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,3584,0.0990943988164266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,51200,1.0930933634440103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,10240,0.22533440589904785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,3072,0.09922346274058023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,8192,0.18194986979166666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,2560,0.07722133000691732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,7168,0.18638827006022135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,3072,0.14519999821980795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,2048,0.11375359694163005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,6144,0.13829439481099445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,2048,0.06332693497339884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,1536,0.07500053246815999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,2048,0.05761706829071045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,1536,0.052190931638081875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,1024,0.05118079980214437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,1024,0.03997333447138469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,2560,0.14209386507670085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,1024,0.02890666723251343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,768,0.046114134788513186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,768,0.03425920009613037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,512,0.027672533194224042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,512,0.027616000175476073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,512,0.017214934031168617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,256,0.015541332960128783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,256,0.021091200908025107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,4096,0.09457066853841146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,128,0.010281599561373393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,128,0.018348799149195353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,3584,0.0960757335027059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,64,0.006589866677920024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,3072,0.07278292973836263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,64,0.0165802667538325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,65536,32,0.004612266520659129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,65536,32,0.016502400239308677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,2560,0.062208000818888345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,1536,0.040216533342997234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,768,0.023229867219924927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,65536,1.2057652791341147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,256,0.012152533729871113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,65536,128,0.009525332848230999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,65536,1.155797322591146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,16384,0.31864426930745443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,65536,2.357544453938802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,16384,0.6156319936116537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,16384,0.2959775924682617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,51200,0.9538911819458008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,12288,0.24101759592692057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,12288,0.48737281163533525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,10240,0.3860042572021484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,10240,0.20208427111307778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,8192,0.18894294102986653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,51200,1.8601632436116535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,8192,0.3057728131612142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,8192,0.17482026418050128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,7168,0.2575477282206217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,7168,0.14627307256062824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,6144,0.23542613983154298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,6144,0.1268064022064209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,7168,0.15409493446350098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,5120,0.18812799453735352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,5120,0.10943679809570313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,4096,0.09103893438975016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,4096,0.1761504014333089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,4096,0.079257599512736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,3584,0.13299307028452556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,3584,0.09042346477508545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,3584,0.0704202651977539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,3072,0.11310293674468994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,2560,0.09450986385345458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,3072,0.07981546719868979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,2560,0.0520362655321757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,2560,0.06900160312652588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,2048,0.0806442658106486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,2048,0.053109331925710046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,1536,0.06014933188756307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,1536,0.04332053263982137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,51200,0.9065322875976562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,1024,0.0461514671643575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,12288,0.2602442741394043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,10240,0.2174762725830078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,1024,0.036817065874735516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,768,0.03565546671549479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,768,0.03123626708984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,512,0.022294400135676067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,512,0.02411093314488729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,512,0.01511253317197164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,256,0.012308266758918763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,256,0.01925119956334432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,256,0.010844799876213073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,5120,0.09746666749318442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,128,0.007719466586907704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,128,0.015727999806404113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,64,0.00448639988899231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,64,0.01611199975013733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,51200,32,0.004287999868392944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,51200,32,0.0160671999057134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,3072,0.061196800072987875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,2048,0.04304320017496745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,6144,0.1159285306930542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,1536,0.03422613143920898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,65536,0.3749504089355469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,65536,0.741865603129069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,16384,0.21789973576863608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,51200,0.2996618588765462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,51200,0.5834112167358398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,16384,0.10709439913431804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,12288,0.08447146415710449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,12288,0.1637898604075114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,1024,0.027268266677856444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,10240,0.11586559613545735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,768,0.01997440059979757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,10240,0.0728991985321045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,8192,0.0945354700088501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,8192,0.06200746695200602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,51200,128,0.008239999910195668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,7168,0.08253866831461588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,7168,0.05594240029652914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,7168,0.04462186495463054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,6144,0.0720415989557902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,6144,0.05023146470387777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,65536,0.3564213434855143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,5120,0.0633354663848877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,5120,0.04433706601460775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,51200,0.27951361338297526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,4096,0.05735573371251425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,16384,0.10958293279012044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,4096,0.038650667667388915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,3584,0.04346346855163574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,3584,0.03572800159454346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,12288,0.0722218672434489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,3072,0.043961600462595625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,10240,0.06132373412450155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,3072,0.03291093309720357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,2560,0.03223466674486796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,2560,0.030086400111516316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,8192,0.050069332122802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,2048,0.026250666379928587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,2048,0.02681066592534383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,1536,0.02068906625111898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,1536,0.0235317329565684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,6144,0.03927893241246541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,1024,0.014679466684659323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,1024,0.020333866278330483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,1024,0.010659199953079224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,768,0.011575466394424439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,768,0.01928106745084127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,768,0.00928533375263214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,512,0.00862506628036499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,512,0.017590399583180746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,5120,0.033717334270477295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,256,0.005585066477457682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,256,0.015098667144775391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,4096,0.03176640073458354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,128,0.003532800078392029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,128,0.015035733580589294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,3584,0.025613866249720257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,64,0.0032042667269706728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,64,0.014987732966740927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,16384,32,0.0033141332368055976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,3072,0.025407999753952026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,16384,32,0.015038933356602988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,2560,0.020105600357055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,2048,0.016711467504501344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,65536,0.29583253860473635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,1536,0.013587199648221335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,65536,0.5559488296508789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,51200,0.4366389274597168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,16384,0.169868803024292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,51200,0.23612906138102213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,16384,0.09754986763000488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,12288,0.11125226815541585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,12288,0.0679744005203247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,10240,0.09310719966888428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,10240,0.05899306535720825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,512,0.007477333148320516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,8192,0.07626026471455892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,8192,0.05198826789855957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,256,0.005319466690222422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,7168,0.06663466691970825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,7168,0.04722559849421183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,16384,128,0.004993066688378652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,6144,0.05737706820170084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,6144,0.04257706801096599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,65536,0.28382186889648436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,5120,0.05484906832377116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,5120,0.038464001814524335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,51200,0.22055360476175942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,16384,0.0761087973912557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,4096,0.04509653250376384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,4096,0.033872000376383465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,4096,0.023155200481414794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,3584,0.035197865962982175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,12288,0.05763733386993408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,3072,0.03268586595853169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,10240,0.049167998631795246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,8192,0.040516265233357746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,2560,0.025838933388392132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,2560,0.0264575997988383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,2560,0.01707520087560018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,2048,0.021371734142303467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,2048,0.023933867613474526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,2048,0.01383039951324463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,7168,0.03622613350550334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,1536,0.021572266022364298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,3584,0.031195733944574994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,1024,0.012010666728019714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,1024,0.019131733973821004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,3072,0.02937920093536377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,768,0.009732266267140705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,6144,0.03163520097732544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,768,0.018312533696492515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,512,0.007389866809050243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,768,0.008152533570925396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,512,0.015203199783960977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,1536,0.01691306630770365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,256,0.003857066730658213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,256,0.015500799814860026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,5120,0.027552000681559247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,128,0.0034826666116714476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,128,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,3584,0.021202133099238078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,64,0.0031338666876157125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,3072,0.018994132677714028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,12288,32,0.0030901332696278887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,32,0.014710399508476257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,65536,0.48035306930541993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,1536,0.01167680025100708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,65536,0.26022507349650065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,12288,64,0.014900267124176025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,1024,0.00931946635246277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,512,0.005618133147557576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,51200,0.20327253341674806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,51200,0.378492800394694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,16384,0.12540266513824463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,16384,0.07728213469187419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,12288,0.09623359839121501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,12288,0.05949759880701701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,10240,0.07879892985026041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,10240,0.05172160069147745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,8192,0.06565866470336915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,10240,0.04212799866994222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,8192,0.035010135173797606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,8192,0.044275200366973876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,7168,0.053188268343607584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,7168,0.040830934047698976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,256,0.005125333368778229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,6144,0.04685653448104858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,12288,128,0.004971733192602793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,6144,0.03761920134226481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,5120,0.03934400081634522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,5120,0.03348480065663655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,4096,0.03197653293609619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,4096,0.029934932788213093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,65536,0.23898773193359374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,3584,0.03226880033810933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,3584,0.02821439901987712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,3584,0.018658133347829182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,51200,0.1834880034128825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,3072,0.024804266293843587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,3072,0.02784213423728943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,16384,0.0744821310043335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,2560,0.021255467335383096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,2560,0.02400533358256022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,2048,0.017493333419164023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,2048,0.02213333249092102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,12288,0.04866026639938355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,1536,0.01418239971001943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,1536,0.020528000593185425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,7168,0.030983465909957885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,1024,0.01016213297843933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,6144,0.027795199553171796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,1024,0.018567466735839845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,768,0.008424533406893413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,768,0.016965333620707193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,5120,0.023963733514149984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,768,0.00695253312587738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,512,0.0065984000762303666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,512,0.01525759994983673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,256,0.00367253323396047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,256,0.01495039959748586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,4096,0.020759467283884683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,128,0.003239466746648153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,128,0.01495039959748586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,3072,0.016590933005015053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,64,0.003054933249950409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,64,0.014593066771825156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,10240,32,0.003067733347415924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,2560,0.014562132954597472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,10240,32,0.014593066771825156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,2048,0.01267413298288981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,65536,0.19987200101216634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,1536,0.010745599865913391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,65536,0.37518399556477866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,1024,0.008804266651471455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,51200,0.16167893409729003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,16384,0.09239892959594727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,51200,0.2966400146484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,51200,0.15361706415812176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,12288,0.07073280016581217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,16384,0.061220268408457436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,512,0.005676800012588501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,12288,0.055713065465291346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,10240,0.059545600414276124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,256,0.005191466708978017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,10240,0.044392534097035724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,8192,0.04817173480987549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,8192,0.02877440055211385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,8192,0.03838613430658976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,7168,0.04275840123494466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,7168,0.03577066659927368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,6144,0.03769280115763347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,10240,128,0.004875733455022176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,6144,0.03268373409907023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,5120,0.03163839975992839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,5120,0.02983786662419637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,12288,0.04025280078252157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,4096,0.025935999552408856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,4096,0.026342399915059406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,4096,0.017222400506337485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,3584,0.023150932788848878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,3584,0.024996266762415568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,65536,0.18589439392089843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,3072,0.02322453260421753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,3072,0.02370880047480265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,16384,0.05154666503270468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,2560,0.01756160060564677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,2560,0.02190720041592916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,10240,0.03429439862569173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,2048,0.014309333761533103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,7168,0.025889066855112712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,2048,0.0207914670308431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,1536,0.011520000298817952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,2048,0.01097920040289561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,1536,0.01939520041147868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,6144,0.022754132747650146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,1024,0.008609066406885784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,1024,0.007782400151093801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,768,0.007315200070540111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,768,0.015560533603032431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,5120,0.020078933238983153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,512,0.005712000032265982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,512,0.015440000096956888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,256,0.003655466685692469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,256,0.01508799990018209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,256,0.005039999882380167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,128,0.0031925333042939507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,128,0.014777599771817525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,128,0.004830933113892873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,64,0.0029674666623274487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,64,0.014724266529083253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,8192,32,0.0029909332593282064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,32,0.014734933773676554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,3584,0.01564479966958364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,8192,1024,0.01748266617457072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,65536,0.3262325286865234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,65536,0.18478506406148273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,51200,0.2577770709991455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,51200,0.1496938705444336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,3072,0.013891200224558512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,16384,0.0946880022684733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,51200,0.1323850631713867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,16384,0.057684266567230226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,12288,0.04639039834340413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,12288,0.06765546798706054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,10240,0.05538986523946127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,10240,0.04083733161290486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,2560,0.012651733557383218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,8192,0.04543253183364868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,8192,0.03577706813812256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,8192,0.02608533302942912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,1536,0.009786666433016459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,7168,0.03800426721572876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,7168,0.03344853321711223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,6144,0.033583998680114746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,6144,0.031057065725326537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,768,0.006018133461475372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,5120,0.02837973237037659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,5120,0.02797226707140605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,8192,512,0.005553066730499268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,4096,0.023485867182413737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,4096,0.025269333521525068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,4096,0.015544533729553223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,3584,0.020911999543507895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,3584,0.024493867158889772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,65536,0.1639839967091878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,3072,0.020807466904322305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,3072,0.023265065749486287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,16384,0.04591466585795085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,2560,0.015794133146603904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,12288,0.03636906544367473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,2560,0.02167466680208842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,2048,0.013086932897567748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,2048,0.02026026646296183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,10240,0.030949334303538006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,1536,0.010725333293279012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,1536,0.018979199727376304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,1536,0.009186133742332458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,1024,0.008170666793982189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,7168,0.0233514666557312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,1024,0.016361600160598753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,768,0.00693333347638448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,1024,0.006930133203665416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,768,0.015410133202870688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,512,0.004645333190759023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,512,0.015117866794268289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,6144,0.02099519968032837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,256,0.0033941333492596946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,256,0.014948266744613647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,5120,0.018372267484664917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,128,0.0031370667119820913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,128,0.01461120049158732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,3584,0.01437013347943624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,64,0.002917333443959554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,3072,0.012613333264986672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,64,0.014661332964897156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,7168,32,0.002994133283694585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,7168,32,0.014622933665911355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,2560,0.011830400427182515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,2048,0.010227200388908387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,65536,0.15846506754557294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,65536,0.28515733083089195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,51200,0.1285141309102376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,51200,0.2233674685160319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,768,0.005987200140953064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,16384,0.07578879992167155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,16384,0.052002131938934326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,16384,0.041111465295155844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,12288,0.05818880001703898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,512,0.005613866448402405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,12288,0.04234346548716227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,12288,0.03267306685447693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,256,0.005129600067933401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,10240,0.037598931789398195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,8192,0.039451734224955244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,8192,0.03373119831085205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,8192,0.02315733234087626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,7168,0.03377600113550822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,7168,0.031652265787124635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,6144,0.030213334163029987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,6144,0.029269333680470782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,5120,0.025518933931986492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,5120,0.026894932985305785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,10240,0.04975359837214152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,4096,0.0215445339679718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,10240,0.02795093258221944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,4096,0.02444266676902771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,3584,0.01924053430557251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,7168,128,0.004924799998601278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,3584,0.023144533236821495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,3072,0.017042134205500284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,3072,0.021824000279108684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,3072,0.011754666765530903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,2560,0.014615466197331747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,2560,0.020644267400105797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,2560,0.010897066195805867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,2048,0.01236799955368042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,2048,0.019437867403030395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,2048,0.009617066383361817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,1536,0.010054399569829304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,65536,0.143395201365153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,1536,0.017988266547520955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,1024,0.007563733557860057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,1024,0.015271466970443726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,51200,0.11458773612976074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,768,0.007049599786599477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,768,0.015270400047302245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,3584,0.013146666685740152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,512,0.003920000046491623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,768,0.005595733225345611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,512,0.015130666891733804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,256,0.0034304000437259674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,256,0.014842666188875833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,256,0.005117866893609365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,128,0.0033183999359607695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,7168,0.02109760046005249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,128,0.01461013356844584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,128,0.004916266600290934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,64,0.002963199963172277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,6144,0.018962132930755615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,6144,32,0.00288426677385966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,64,0.014596266547838846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,6144,32,0.014615466197331747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,65536,0.13976532618204754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,65536,0.24191573460896812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,51200,0.19116053581237794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,51200,0.10849599838256836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,16384,0.063591468334198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,51200,0.09954346815745035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,16384,0.0458954652150472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,12288,0.048979198932647704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,4096,0.013988266388575235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,5120,0.016401066382726034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,12288,0.037780265013376876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,10240,0.04169706503550212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,12288,0.028691200415293376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,10240,0.03363519906997681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,8192,0.033885868390401204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,8192,0.030491733551025392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,7168,0.028165332476298016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,8192,0.02060799996058146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,7168,0.028148265679677327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,6144,0.024870399634043375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,6144,0.026321067412694292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,6144,0.016923733552296958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,5120,0.021386667092641195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,5120,0.024306132396062215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,1536,0.008693333466847737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,4096,0.01776640017827352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,1024,0.00629013329744339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,4096,0.022382932901382446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,3584,0.015955199797948204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,3584,0.02185493310292562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,3584,0.012085333466529846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,6144,512,0.00528106689453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,3072,0.01397333343823751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,3072,0.020942932367324828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,3072,0.01095360020796458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,2560,0.012170666456222534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,2560,0.020080000162124634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,2560,0.010413866241772969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,2048,0.010393599669138592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,2048,0.018489599227905273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,65536,0.12522559960683186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,1536,0.009546666344006857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,1536,0.016458666324615477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,16384,0.03574826717376709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,1024,0.006826666494210561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,1024,0.0155349334081014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,10240,0.02472533384958903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,768,0.005435733497142792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,768,0.01569919983545939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,7168,0.01909546653429667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,768,0.005765333275000254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,512,0.0036490666369597114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,512,0.015029333035151162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,256,0.003311999887228012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,256,0.01483519971370697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,5120,0.014762666821479798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,128,0.0030965333183606463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,128,0.014613333344459533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,4096,0.012621866663297019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,64,0.002811733384927114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,64,0.01458453337351481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,5120,32,0.0028714666763941447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,5120,32,0.014430933197339377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,65536,0.11039040088653565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,65536,0.10728533267974853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,51200,0.14707627296447753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,2048,0.008968533078829447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,1536,0.007298133273919423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,16384,0.05035839875539144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,16384,0.03972906668980916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,16384,0.03153706590334575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,12288,0.03959466616312663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,12288,0.03405333360036214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,1024,0.006277333199977875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,10240,0.033617067337036136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,10240,0.030817067623138426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,512,0.005348266661167144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,51200,0.09008320172627768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,8192,0.027958399057388304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,8192,0.027506132920583088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,7168,0.025063467025756837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,256,0.005075199902057648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,7168,0.025867732365926106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,6144,0.022300799687703453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,6144,0.024130133787790935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,5120,128,0.004878933231035868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,5120,0.019550933440526327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,5120,0.022977066040039063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,65536,0.18568746248881024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,4096,0.01803626616795858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,4096,0.020829866329828896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,4096,0.011683199803034465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,3584,0.014387200276056925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,3584,0.02055466572443644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,51200,0.08555839856465658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,3072,0.012811733285586038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,3072,0.01955946683883667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,3072,0.010133333007494609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,2560,0.010461866855621338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,2560,0.018652800718943277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,2560,0.009544533491134644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,2048,0.00902826686700185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,2048,0.017492266496022542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,12288,0.025270400444666545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,1536,0.007562666634718577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,1536,0.015546666582425437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,1536,0.007166933516661327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,10240,0.021941334009170532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,1024,0.005975466469923655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,1024,0.015155200163523355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,768,0.004130133241415024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,768,0.015146666765213012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,768,0.005545599758625031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,8192,0.01804693341255188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,512,0.0035317334036032355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,512,0.01488746702671051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,256,0.0031669333577156065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,256,0.014621866742769876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,256,0.005011199911435445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,128,0.002932266642649968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,7168,0.016824533541997276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,128,0.014509866635004679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,6144,0.01492693324883779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,64,0.0028512001037597655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,4096,32,0.003094399968783061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,64,0.014636799693107605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,4096,32,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,5120,0.013347199559211731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,65536,0.16989866892496747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,65536,0.1037397305170695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,65536,0.10268692970275879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,51200,0.1342037359873454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,51200,0.0851797342300415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,16384,0.048631465435028075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,51200,0.08225706418355307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,16384,0.0367850661277771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,3584,0.011150933305422465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,12288,0.03480000098546346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,12288,0.03157013257344564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,10240,0.03001813292503357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,10240,0.028833067417144774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,2048,0.008362666765848795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,8192,0.02541973392168681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,8192,0.02600533366203308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,16384,0.03004693388938904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,8192,0.016884267330169678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,7168,0.022503467400868733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,7168,0.024306132396062215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,6144,0.0203274667263031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,1024,0.005940266450246175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,6144,0.02308053374290466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,5120,0.017565866311391197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,5120,0.02170133392016093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,512,0.005231999854246775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,4096,0.014686933159828186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,4096,0.02062293291091919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,4096,0.011575466394424439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,3584,0.013289599617322286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,3584,0.020115200678507486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,4096,128,0.004816000163555145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,3584,0.010965333382288615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,3072,0.010908800363540649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,3072,0.019166932503382365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,3072,0.010013866424560546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,2560,0.009703466296195984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,2560,0.018649599949518838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,2048,0.008381866415341695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,2048,0.016445866227149962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,1536,0.007117866476376851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,1536,0.015689599514007568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,12288,0.024194133281707764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,1024,0.004986666639645894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,1024,0.015318399667739869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,10240,0.020973867177963255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,768,0.0038474666575590765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,768,0.015043200055758158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,7168,0.01612160007158915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,512,0.0035594666997591654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,6144,0.01458560029665629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,512,0.014899200201034546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,256,0.0031914666295051576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,256,0.014674133062362671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,256,0.005009066561857859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,5120,0.01267519990603129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,128,0.0028736000259717304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,128,0.014430933197339377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,64,0.002842666705449422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,64,0.014321066935857139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3584,32,0.0028970666229724885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3584,32,0.014334932963053385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,65536,0.14665385882059734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,2560,0.009026133020718892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,65536,0.0900544007619222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,51200,0.11542399724324544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,2048,0.007522133489449819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,51200,0.07283519903818766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,51200,0.0759231964747111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,1536,0.006866133213043213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,16384,0.04018453359603882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,16384,0.03496319850285848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,12288,0.03188693324724833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,12288,0.03046826720237732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,10240,0.027211733659108478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,1024,0.0061034664511680605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,10240,0.02735146681467692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,8192,0.023294933636983237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,8192,0.02528746724128723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,768,0.005635199944178263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,8192,0.016698666413625083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,7168,0.020836265881856282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,7168,0.023964800437291465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,6144,0.01844266653060913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,6144,0.02249706586201986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,16384,0.02809813419977824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,5120,0.01609493295351664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,5120,0.02113386591275533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,512,0.005273599922657013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,5120,0.012570666273434958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,4096,0.013885866602261862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,4096,0.019862399498621622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,4096,0.010856533050537109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,3584,0.01276479959487915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,3584,0.019403733809789023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,3584,0.010564266641934713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3584,128,0.004859733581542969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,3072,0.0114464004834493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,3072,0.00962773362795512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,3072,0.018739199638366698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,2560,0.008756267031033833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,2560,0.01764906644821167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,2048,0.007838933169841767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,2048,0.016296533743540446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,65536,0.09545599619547526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,1536,0.006782933572928111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,1536,0.015954132874806723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,1536,0.006572799881299336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,1024,0.0045973335703214015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,1024,0.015212800105412802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,1024,0.005916800101598104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,768,0.0038602667550245917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,12288,0.021901865800221763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,768,0.01514240006605784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,512,0.0034986667335033415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,2560,0.009048533439636231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,10240,0.01956693331400553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,512,0.014868266383806863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,256,0.003223466624816259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,512,0.0052704001466433205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,256,0.00489279975493749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,128,0.0029343999922275543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,256,0.014709333578745524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,64,0.00276799996693929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,128,0.014575999975204468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,3072,32,0.0028213332096735638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,64,0.0145087997118632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,7168,0.015293866395950317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,3072,32,0.014620799819628397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,65536,0.12122560342152913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,65536,0.07766079902648926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,65536,0.09360106786092123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,51200,0.09675626754760742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,6144,0.013925333817799887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,16384,0.033562668164571124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,16384,0.03128426671028137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,12288,0.02672959963480632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,12288,0.0272053341070811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,12288,0.022036266326904298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,10240,0.023388799031575522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,10240,0.024808533986409507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,8192,0.019350399573644005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,2048,0.007369600236415863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,51200,0.06292266845703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,8192,0.023052799701690673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,7168,0.0175327996412913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,7168,0.02230400045712789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,6144,0.015507200360298156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,6144,0.021076265970865884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,6144,0.013765333096186319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,5120,0.013666133085886637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,5120,0.020310399929682414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,5120,0.01237440009911855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,4096,0.011912533640861511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,4096,0.01939199964205424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,4096,0.010937600334485372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,3584,0.010987733801205952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,3584,0.018961066007614137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,768,0.005590400099754334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,3072,0.010136533776919048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,3072,0.01776640017827352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,3072,0.008936533331871032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,2560,0.008251733581225077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,3072,128,0.004825599988301595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,2560,0.016687999169031777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,2048,0.007162666817506154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,8192,0.01621333360671997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,2048,0.016167466839154564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,1536,0.005717333157857259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,1536,0.016064000129699708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,51200,0.07410240173339844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,1024,0.004250666499137879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,16384,0.02744106650352478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,1024,0.015278933445612588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,768,0.003912533322970072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,1024,0.005942399799823761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,768,0.015123200416564942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,512,0.0035573333501815797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,512,0.014978133638699851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,10240,0.01914026737213135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,256,0.0032000000278155005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,256,0.014605866869290671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,7168,0.015096533298492431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,128,0.002976000060637792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,128,0.014481066664059957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,3584,0.010430933038393656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,64,0.0028192001084486645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,64,0.014596266547838846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2560,32,0.002872533351182938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2560,32,0.014626133441925048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,2560,0.008081066608428954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,65536,0.09681706428527832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,2048,0.007063466807206471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,65536,0.06400746504465739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,51200,0.0758506695429484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,65536,0.08570666313171386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,51200,0.053683201471964516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,16384,0.02762666742006938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,1536,0.006502399841944377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,16384,0.027690666913986205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,16384,0.025517867008845015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,12288,0.024336000283559166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,12288,0.020457599560419717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,10240,0.01924906571706136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,768,0.005508266886075338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,10240,0.023031467199325563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,8192,0.01609173317750295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,8192,0.021530665953954062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,512,0.005312000215053558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,8192,0.01548479994138082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,7168,0.014498133460680643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,7168,0.02065920035044352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,6144,0.012972799936930337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,6144,0.019972266753514607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,256,0.0050005331635475155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,5120,0.011780266960461933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,5120,0.019964800278345744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,12288,0.022155733903249104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,4096,0.010193066795667012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,4096,0.01859626571337382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,4096,0.010178132851918539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,3584,0.00965226689974467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,3584,0.0173632005850474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2560,128,0.004882133503754934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,3072,0.008954667051633199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,51200,0.06809919675191244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,3072,0.016721065839131674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,2560,0.00767146646976471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,2560,0.01646080017089844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,2048,0.006674133241176605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,10240,0.01797973314921061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,2048,0.015947733322779337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,1536,0.004806399842103322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,1536,0.015608533223470052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,7168,0.014364799857139588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,1024,0.004145066688458125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,1024,0.01511146624883016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,6144,0.012939733266830445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,1024,0.0057205334305763245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,768,0.0037770666182041167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,768,0.015072000026702882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,512,0.003416533271471659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,512,0.014972800016403198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,5120,0.011707733074824016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,256,0.0031285333136717476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,256,0.014723199605941772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,256,0.004955733319123586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,128,0.0028309332827727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,128,0.014413866400718688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,128,0.004725333551565806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,64,0.0027583998938401537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,64,0.014442666371663412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,3584,0.009245866537094116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,2048,32,0.0028053333361943563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,2048,32,0.01451520025730133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,65536,0.07493653297424316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,65536,0.05366933345794678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,51200,0.05958400170008341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,51200,0.04527466694513957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,51200,0.06718080043792725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,16384,0.022539732853571574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,16384,0.024650667111078897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,16384,0.025073067347208662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,12288,0.018221867084503175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,12288,0.022361600399017335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,12288,0.020193066199620566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,10240,0.015545599659283958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,10240,0.021413334210713706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,3072,0.008789333701133727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,8192,0.013313066959381104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,8192,0.020159999529520668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,2560,0.007961600025494894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,2048,0.006981333096822103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,8192,0.015052800377209982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,7168,0.012314666310946147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,7168,0.020413867632548013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,6144,0.011373866597811382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,6144,0.019156267245610557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,5120,0.01000853379567464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,5120,0.019019732872645058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,1536,0.006568533182144165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,5120,0.011082667112350463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,4096,0.017230933904647826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,4096,0.00944533348083496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,768,0.005448533097902933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,3584,0.008657067020734151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,3584,0.016978132724761962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,3072,0.008098133405049642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,2048,512,0.005231999854246775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,3072,0.016804265975952148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,2560,0.0069365332523981735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,2560,0.016581333676973977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,4096,0.009143466750780743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,2048,0.00543146679798762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,2048,0.01606826682885488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,65536,0.0844170649846395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,1536,0.004808533191680908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,1536,0.015562666455904641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,10240,0.01776319940884908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,1024,0.004057600100835165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,1024,0.015212800105412802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,7168,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,6144,0.01283093293507894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,768,0.003702399879693985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,768,0.014878933628400167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,3584,0.009065600236256917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,512,0.0034634667138258614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,512,0.014963199694951376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,3072,0.008157866696516674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,512,0.005186133086681366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,256,0.003053866575161616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,256,0.014707199732462563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,256,0.0048981333772341405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,128,0.0028959999481836954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,2560,0.007639466722806294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,128,0.014401066303253173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,64,0.002677333354949951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1536,32,0.002757333219051361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,32,0.014487466216087342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,65536,0.05041706562042236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,65536,0.041713066895802814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,2048,0.006853333115577698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,51200,0.04074560006459554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,51200,0.03642559846242269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,1536,0.006554666658242543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,16384,0.016220800081888833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,128,0.004740266501903534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,16384,0.021703465779622396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1536,64,0.014429866274197897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,12288,0.019942400852839152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,1024,0.005737600227197012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,10240,0.012025599678357441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,10240,0.01962453325589498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,8192,0.010533333818117777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,8192,0.01907093326250712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,51200,0.06696000099182128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,7168,0.00985599954922994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,7168,0.018618667125701906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,12288,0.013529599706331889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,6144,0.008834133545557659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,6144,0.017326933145523072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,65536,0.08414826393127442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,5120,0.008454400300979614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,5120,0.01835306684176127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,16384,0.02502080003420512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,4096,0.008091733356316884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,12288,0.01995413303375244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,4096,0.017340799172719322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,3584,0.007210666437943776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,3584,0.01684266726175944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1536,768,0.005463466544946035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,3072,0.006495999793211619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,3072,0.016646400094032288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,3072,0.008041599889596303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,10240,0.017576533555984496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,2560,0.005997866888840993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,2560,0.016394666830698647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,8192,0.014938666423161825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,2048,0.005406933526198069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,2048,0.015871999661127727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,1536,0.004653866589069367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,1536,0.015404799580574035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,1536,0.006412800153096516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,1024,0.004072533299525579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,1024,0.015191466609636942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,7168,0.013474133610725404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,768,0.0036821333070596062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,768,0.01509119967619578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,6144,0.011988266309102377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,512,0.0033269333342711128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,512,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,512,0.005234133203824361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,256,0.0030645333230495454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,256,0.014552533626556396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,256,0.004840533435344696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,128,0.002825599908828735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,128,0.014436266819636025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,5120,0.010542933146158855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,64,0.0026880001028378804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,64,0.014591999848683677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,1024,32,0.002665599932273229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,1024,32,0.014443733294804893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,65536,0.0397216002146403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,65536,0.03684266805648804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,65536,0.08416960239410401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,51200,0.032434133688608806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,51200,0.03195306658744812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,4096,0.009358933568000794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,16384,0.013426132996877036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,3584,0.00900266667207082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,16384,0.020654932657877604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,12288,0.011598933736483257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,12288,0.019341866175333657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,2560,0.0076906666159629825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,10240,0.010524800419807434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,2048,0.006878933310508728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,10240,0.01837546626726786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,10240,0.017179733514785765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,8192,0.009348266323407491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,8192,0.017579734325408936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,7168,0.008693333466847737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,7168,0.018173867464065553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,7168,0.013404800494511922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,1024,0.005762133498986562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,6144,0.008286933104197185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,6144,0.01784320076306661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,768,0.0054293334484100345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,5120,0.007269333302974701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,5120,0.018654932578404747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,1024,128,0.0046528001626332605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,4096,0.006362666686375936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,4096,0.017926400899887084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,4096,0.009318400422732036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,3584,0.006680533289909363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,3584,0.01723519961039225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,51200,0.06678720315297446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,3072,0.006121600170930227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,3072,0.016769067446390788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,3072,0.008142933249473572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,2560,0.0061141331990559895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,2560,0.01665386656920115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,16384,0.024821333090464272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,2048,0.005489066739877065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,12288,0.019817600647608437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,2048,0.006859733164310456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,1536,0.004786133269468943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,1536,0.015474133690198264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,8192,0.014375467101732889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,1024,0.004035199930270513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,1024,0.015095466375350952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,768,0.003718400001525879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,768,0.015102932850519816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,768,0.005398400127887726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,6144,0.011947733163833619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,512,0.0034154665966828666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,5120,0.010484266281127929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,512,0.00521919975678126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,512,0.014806399742762247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,2560,0.007684266567230225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,256,0.0030517332255840302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,128,0.0029056000212828318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,256,0.00494400014479955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,256,0.014620799819628397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,128,0.01470080018043518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,64,0.002657066782315572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,128,0.004769066472848257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,768,32,0.002678400029738744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,64,0.014598400394121806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,32,0.01455466647942861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,65536,0.028175999720891316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,65536,0.029277867078781127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,51200,0.023334399859110514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,51200,0.026190932591756182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,16384,0.010617599884668986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,16384,0.018837332725524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,12288,0.009302399555842082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,12288,0.017493333419164023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,3584,0.008900266885757447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,10240,0.00862613320350647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,10240,0.01744426687558492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,768,2048,0.01607253352801005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,8192,0.00784853349129359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,8192,0.017438934246699015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,1536,0.00653546651204427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,7168,0.006640000144640605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,7168,0.018037333091100057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,768,1024,0.005762133498986562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,6144,0.006342400113741558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,6144,0.017502933740615845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,6144,0.01160426636536916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,5120,0.006355200211207073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,5120,0.017972266674041747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,65536,0.08411733309427896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,51200,0.06676479975382486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,4096,0.005949866771697998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,4096,0.01758079926172892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,3584,0.006457599997520447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,12288,0.01965226729710897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,3584,0.017105066776275636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,3072,0.006110933423042297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,3072,0.017146666844685875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,3072,0.00798933357000351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,2560,0.006140799820423126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,2560,0.01650773286819458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,10240,0.017126399278640746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,2048,0.0054666668176651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,2048,0.015874133507410685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,8192,0.014265599846839904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,16384,0.02448106606801351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,1536,0.004818133513132731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,1536,0.01548479994138082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,1024,0.004067199925581614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,1024,0.015238400300343832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,7168,0.013044266899426778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,768,0.0038389332592487337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,768,0.014969600240389505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,5120,0.010524800419807434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,512,0.0033930666744709016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,512,0.014859732985496522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,512,0.0049685334165891016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,256,0.00311253344019254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,256,0.014523733655611673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,4096,0.009330133597056072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,3584,0.008921600381533305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,128,0.002881066749493281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,128,0.01458453337351481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,2560,0.007653333246707916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,64,0.0026869334280490874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,64,0.014644267161687216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,512,32,0.0027072000006834666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,512,32,0.014792533715566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,65536,0.017467733224232992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,65536,0.023922133445739745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,2048,0.006818133095900218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,51200,0.013889066378275552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,1536,0.006483200192451477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,51200,0.022215465704600014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,16384,0.007851733267307282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,1024,0.005686399837334951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,16384,0.01820053259531657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,12288,0.006454400221506755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,16384,0.024036266406377158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,12288,0.01764906644821167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,10240,0.006381866832574208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,10240,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,768,0.005406933526198069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,8192,0.006257066627343495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,8192,0.017432532707850137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,8192,0.014094932874043783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,7168,0.006106666723887126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,7168,0.017903999487559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,7168,0.01301653285821279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,6144,0.005921066800753275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,6144,0.01761173407236735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,6144,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,5120,0.00631466656923294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,5120,0.017896533012390137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,256,0.004844800134499868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,4096,0.00588266650835673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,4096,0.018308266003926595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,4096,0.00920746624469757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,3584,0.0063381334145863845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,3584,0.016898133357365928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,3584,0.008703999718030294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,3072,0.006018133461475372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,512,128,0.004678399860858917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,2560,0.006002133091290792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,3072,0.016548267006874083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,2560,0.01656000018119812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,2048,0.005310933291912079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,2048,0.016100266575813295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,65536,0.08396373589833578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,1536,0.004712533454100291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,1536,0.015656532843907674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,51200,0.06664426724116007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,1024,0.004084266722202301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,1024,0.01503679951032003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,12288,0.01908586621284485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,768,0.0036874666810035707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,768,0.014807466665903726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,768,0.005460266768932342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,512,0.003332266708215078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,10240,0.016696532567342125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,5120,0.010517332951227825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,256,0.0030720000465710956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,256,0.014607999722162882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,256,0.004844800134499868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,128,0.002754133443037669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,1024,0.005771733323733012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,128,0.014485333363215128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,64,0.002624000112215678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,128,0.0046965335806210835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,256,32,0.002665599932273229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,64,0.014516266187032065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,65536,0.011901866396268208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,32,0.014432000120480857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,65536,0.020862932999928793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,51200,0.01025279959042867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,256,512,0.014656000336011252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,3072,0.008001066744327545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,51200,0.019316265980402626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,16384,0.01779200037320455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,2560,0.0075530668099721265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,12288,0.006179200112819671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,2048,0.006742399930953979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,12288,0.017429333925247193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,12288,0.01911466717720032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,10240,0.006099199752012888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,10240,0.01783039967219035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,8192,0.0060703997810681665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,8192,0.01758506695429484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,1536,0.006409599880377452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,7168,0.005940266450246175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,7168,0.018024533987045288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,16384,0.005964800218741099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,6144,0.005779199798901876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,6144,0.017107200622558594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,6144,0.011587199568748475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,5120,0.006117333471775055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,5120,0.017500799894332886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,10240,0.01672640045483907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,256,512,0.0049781332413355505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,4096,0.00573333352804184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,4096,0.017505067586898803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,4096,0.009313066800435383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,3584,0.0061749334136645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,3584,0.016838399569193523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,65536,0.08332160313924154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,3072,0.005827199916044871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,51200,0.06648319959640503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,3072,0.01654293338457743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,2560,0.005771733323733012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,16384,0.023925334215164185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,2560,0.016484266519546507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,2048,0.0051584000388781226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,2048,0.015808000167210897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,8192,0.01402666668097178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,1536,0.0045962666471799215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,1536,0.015717333555221556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,7168,0.012970667084058127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,1024,0.003946666667858759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,1024,0.014924800395965577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,1024,0.005671466886997223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,768,0.0036629334092140196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,768,0.014885333180427552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,5120,0.010429867108662923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,512,0.003319466610749563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,512,0.014693333705266317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,512,0.005049600203831991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,256,0.0030271999537944795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,256,0.01442026694615682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,128,0.0028309332827727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,3584,0.008756267031033833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,128,0.01423893372217814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,128,0.004733866453170777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,64,0.0026506667335828146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,128,32,0.0026634665826956432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,64,0.014333867033322654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,65536,0.008804266651471455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,128,32,0.01418346663316091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,51200,0.007413333157698314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,65536,0.019665066401163736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,16384,0.005901866654555003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,51200,0.01909866730372111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,12288,0.005843199789524078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,16384,0.018218666315078735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,10240,0.006049066781997681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,10240,0.01768959959348043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,8192,0.005884799857934316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,8192,0.01741546591122945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,7168,0.005730133255322774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,7168,0.018024533987045288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,3072,0.007977599898974102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,6144,0.017395200332005818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,2560,0.007623466849327088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,5120,0.006086400151252747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,4096,0.0056757330894470215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,5120,0.01755626598993937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,3584,0.006080000102519989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,4096,0.01690346598625183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,3072,0.005793066819508871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,3072,0.016201600432395935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,2560,0.005613866448402405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,2048,0.006790400048096975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,2048,0.005123200019200643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,2560,0.01593280037244161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,1536,0.00446720023949941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,2048,0.01563093364238739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,1024,0.003902933249870936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,1536,0.015219199657440185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,1536,0.006320000191529592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,768,0.0035402665535608927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,1024,0.015016532937685647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,512,0.0031946666538715364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,768,0.014949333667755128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,256,0.002856533229351044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,512,0.01472106675306956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,128,0.0027445333699385325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,256,0.014573867122332254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,64,0.00266239990790685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,128,0.014327466487884521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,32,0.0026517334083716077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,64,0.014451199769973755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,65536,0.008946133653322856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,32,0.01430400013923645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,12288,0.01722773313522339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,51200,0.007683200140794118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,65536,0.019530665874481202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,16384,0.005789866546789805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,51200,0.018665599822998046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,12288,0.0058037335673968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,16384,0.01770133376121521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,10240,0.006052266558011373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,8192,0.005897599955399831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,10240,0.017461333672205606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,7168,0.0057322666049003605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,8192,0.017259732882181803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,6144,0.005610666672388713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,7168,0.017746132612228394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,5120,0.005962666869163513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,768,0.005423999826113383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,6144,0.017378133535385133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,4096,0.005526400109132131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,5120,0.01760853330294291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,3584,0.0060693333546320595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,4096,0.016942934195200602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,3072,0.0057322666049003605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,64,6144,0.005621333420276642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,3584,0.016829866170883178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,2560,0.005707733333110809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,3072,0.016501333316167197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,2048,0.004995200037956238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,1536,0.004443733394145966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,2560,0.016165332992871602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,1024,0.0038421332836151125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,1536,0.015454933047294617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,768,0.0035562666753927866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,1024,0.015001599987347921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,512,0.003206400076548258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,768,0.014728533228238425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,256,0.002863999952872594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,64,3584,0.01711146632830302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,128,0.002726399898529053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,256,0.01432213286558787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,64,0.0026464000344276427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,128,0.01430293321609497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,13,32,32,0.00264533335963885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,64,0.01443839967250824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,32,0.014330666263898215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,13,128,256,0.0048437332113583885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,12288,0.017262933651606242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,2048,0.015749333302179973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,13,32,512,0.014791466792424521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,16384,0.747592544555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,16384,0.3935551961263021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,12288,0.26977812449137367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,12288,0.5668085098266602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,12288,0.29747734069824217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,51200,1.1815605163574219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,51200,1.0930421193440756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,10240,0.25689172744750977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,10240,0.4731594721476237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,8192,0.24074346224466958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,8192,0.4342133204142253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,51200,2.339991505940755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,7168,0.1827125390370687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,6144,0.28545811971028645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,6144,0.13805012702941893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,6144,0.18246933619181316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,5120,0.23790507316589354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,5120,0.13379093805948894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,4096,0.19235733350118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,4096,0.10960426330566406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,4096,0.10708159605662029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,3584,0.168068265914917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,3584,0.09864319960276285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,7168,0.3326165199279785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,3072,0.14463680585225422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,3072,0.08945600191752115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,2560,0.12171520392100017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,16384,0.3567061424255371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,2560,0.07525119781494141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,2560,0.06456640164057413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,2048,0.09762667020161947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,2048,0.06337386767069499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,1536,0.07501546541849771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,2048,0.05087893406550089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,1536,0.05213760137557984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,1024,0.0508629322052002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,10240,0.2256458600362142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,1024,0.04002453486124675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,768,0.039851733048756915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,768,0.03439040184020996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,8192,0.18144319852193197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,7168,0.15975252787272137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,512,0.03163839975992839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,512,0.029865600665410358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,256,0.01760960022608439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,256,0.021092265844345093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,5120,0.11627093156178792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,128,0.01035093367099762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,128,0.018897066513697304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,3072,0.072762664159139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,3584,0.08375146389007568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,64,0.006275199850400289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,65536,32,0.004628266890843709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,64,0.016338133811950685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,65536,32,0.016404267152150473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,1536,0.0401802659034729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,1024,0.03206613262494405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,768,0.0232149342695872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,512,0.017191465695699057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,256,0.0122079998254776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,65536,1.2062389373779296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,65536,128,0.009569066762924194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,65536,1.1545397440592446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,16384,0.6108874638875326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,16384,0.36954240798950194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,51200,0.9532512029012045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,65536,2.3551615397135417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,51200,0.9055072148640951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,51200,1.8609354654947918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,12288,0.23695786794026694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,12288,0.22340799967447916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,12288,0.46745812098185224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,10240,0.3808608055114746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,10240,0.22998293240865073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,8192,0.16656640370686848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,8192,0.1509877363840739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,8192,0.35654080708821617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,7168,0.14586346944173176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,7168,0.2571221351623535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,6144,0.2231701374053955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,5120,0.18462400436401366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,6144,0.1283456007639567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,5120,0.11468373139699299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,4096,0.15307307243347168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,4096,0.08994026978810629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,3584,0.13828159968058268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,3584,0.08130239645640056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,16384,0.2969792048136393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,3072,0.12059413592020671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,3072,0.07130239804585775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,2560,0.09462186495463053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,10240,0.2174389362335205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,2560,0.06182080109914144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,2048,0.07809173266092936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,2048,0.05328640143076578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,7168,0.13884693781534832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,1536,0.059266134103139245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,1536,0.04381866852442424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,5120,0.10192960103352863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,1024,0.040193065007527666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,6144,0.13269866307576497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,1024,0.03411519924799601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,768,0.03107946713765462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,4096,0.09029226303100586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,768,0.028965334097544353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,512,0.02188906669616699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,512,0.02407360076904297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,3584,0.07076053619384766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,256,0.012604799866676331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,256,0.01931519905726115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,3072,0.061084798971811925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,128,0.007776000102361043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,128,0.015441067020098367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,2560,0.058677331606547034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,64,0.004494933287302653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,2048,0.042771200338999435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,64,0.016149333119392394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,51200,32,0.004296533266703288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,51200,32,0.01619733373324076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,1536,0.03396159807840983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,1024,0.02512213389078776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,768,0.020012799898783365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,65536,0.3754517237345377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,512,0.015412267049153647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,65536,0.7416255950927735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,16384,0.18396053314208985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,16384,0.1232426643371582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,51200,0.5832319895426432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,12288,0.1435754617055257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,12288,0.09710933367411295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,12288,0.0722645362218221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,10240,0.11600853602091472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,10240,0.07245333194732666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,10240,0.06117653449376424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,8192,0.09425813357035319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,8192,0.06147520144780477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,7168,0.08254079818725586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,7168,0.05553493499755859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,128,0.008102400104204814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,6144,0.07167466481526694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,6144,0.04980479876200358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,51200,0.2969738642374674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,6144,0.03913386662801106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,5120,0.07038719654083252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,5120,0.04453440109888713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,4096,0.049250133832295734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,4096,0.03862506548563639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,51200,256,0.011317333579063416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,65536,0.3561962763468424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,3584,0.04354026714960734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,3584,0.039434667428334555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,3072,0.04065066576004028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,16384,0.09438827037811279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,3072,0.03302506605784099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,2560,0.0320362667242686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,51200,0.27957547505696617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,2560,0.029967999458312987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,2048,0.02623680035273234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,2048,0.02672959963480632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,2048,0.016685867309570314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,1536,0.020758400360743202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,1536,0.023489065965016685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,1536,0.013553067048390707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,1024,0.014707199732462563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,1024,0.020602667331695558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,8192,0.05764906803766886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,768,0.01162986656030019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,768,0.019062399864196777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,768,0.009311999877293904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,512,0.008588799834251403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,5120,0.03372373183568318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,512,0.017234132687250773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,4096,0.027911466360092164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,256,0.005623466769854228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,256,0.015194666385650635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,7168,0.04477546612421672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,128,0.0035157332817713416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,128,0.015075199802716575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,128,0.004990933338801066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,64,0.0032245332996050516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,64,0.015050666530927024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,16384,32,0.0032490665713946023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,16384,32,0.015098667144775391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,3584,0.02555519938468933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,3072,0.022523732980092366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,2560,0.02247040073076884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,65536,0.2988650639851888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,65536,0.5545450846354167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,1024,0.010683733224868774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,51200,0.23834667205810547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,16384,0.1449994723002116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,16384,0.0886517365773519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,16384,0.0755850632985433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,12288,0.11211840311686198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,12288,0.07037333647410074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,10240,0.09305600325266519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,512,0.007657599945863088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,8192,0.07621119817097982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,8192,0.05186560153961182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,8192,0.0404853343963623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,16384,256,0.0053951998551686605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,7168,0.06631573438644409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,7168,0.047790932655334475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,6144,0.057366398970286045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,6144,0.0428330659866333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,51200,0.4359840075174968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,5120,0.05466986497243246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,5120,0.04157333374023438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,10240,0.06097919940948486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,4096,0.039527467886606854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,4096,0.03345706860224406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,4096,0.023139200607935586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,3584,0.03497386773427327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,3584,0.03150506615638733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,3584,0.021294933557510377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,3072,0.030374399820963544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,65536,0.2868917465209961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,3072,0.029501867294311524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,2560,0.02592639923095703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,3072,0.021041067441304524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,2560,0.016812799374262492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,2560,0.02656000057856242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,2048,0.020823466777801513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,2048,0.02398186723391215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,1536,0.01693333387374878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,1536,0.02155733307202657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,51200,0.22672746976216634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,1024,0.013249066472053529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,1024,0.019318399826685588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,12288,0.05709760189056397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,768,0.010851200421651204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,10240,0.04879039923350016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,768,0.018023467063903807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,512,0.007496533294518788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,512,0.01532906691233317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,7168,0.036076800028483076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,256,0.0038453333079814913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,256,0.015125333269437154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,6144,0.0321397324403127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,128,0.00342399999499321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,128,0.014798933267593383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,128,0.0048991998036702475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,64,0.0031338666876157125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,64,0.014805333813031516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,12288,32,0.00314026673634847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,5120,0.027756800254185993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,12288,32,0.014723199605941772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,65536,0.2597930590311686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,65536,0.4795712153116862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,2048,0.01402773360411326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,65536,0.2395253340403239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,1536,0.011868799726168316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,51200,0.3780170758565267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,51200,0.20298345883687338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,16384,0.12506240208943684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,16384,0.08597226937611899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,12288,0.09595946470896402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,1024,0.009355733791987102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,12288,0.05923519929250082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,10240,0.07875413099924723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,10240,0.05159146785736084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,8192,0.0652896006902059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,8192,0.044250667095184326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,512,0.005835733314355215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,7168,0.05520106554031372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,7168,0.041788800557454424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,256,0.005234133203824361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,6144,0.046397864818573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,6144,0.037163734436035156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,16384,0.06475520133972168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,6144,0.027698133389155073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,5120,0.039109333356221514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,5120,0.03364693323771159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,5120,0.023949867486953734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,4096,0.03176533381144206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,4096,0.029736532767613726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,12288,768,0.00816319982210795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,4096,0.02059626579284668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,3584,0.028325333197911577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,3584,0.028280532360076903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,3584,0.018611200650533042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,3072,0.024889600276947022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,3072,0.025685334205627443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,3072,0.016598400473594666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,2560,0.021266132593154907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,2560,0.02413439949353536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,2560,0.014751999576886495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,2048,0.01746666630109151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,2048,0.02195733388264974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,1536,0.013852799932161966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,1536,0.020297600825627645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,1536,0.010958932836850484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,1024,0.010150399804115296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,1024,0.018636800845464072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,51200,0.18304959932963055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,768,0.009336533149083455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,768,0.016285866498947144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,12288,0.04950079917907715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,512,0.006621866424878438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,512,0.015124266346295675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,10240,0.0417141318321228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,256,0.0036586667100588477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,256,0.015093333522478738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,256,0.005243733525276184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,128,0.0032138665517171226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,8192,0.03523199955622355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,128,0.014940800269444785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,128,0.004885333279768625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,64,0.014914133151372275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,32,0.0029674666623274487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,7168,0.03083840012550354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,10240,32,0.014847999811172486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,65536,0.2003392060597738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,65536,0.3796565373738607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,51200,0.29846080144246423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,10240,64,0.003005866706371307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,51200,0.14616427421569825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,51200,0.1615957260131836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,16384,0.09236799875895182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,16384,0.061169068018595375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,2048,0.012802132964134216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,12288,0.0707306702931722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,12288,0.0499669353167216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,10240,0.05925866762797037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,10240,0.04437226851781209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,12288,0.04012800057729085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,8192,0.04840426842371623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,1024,0.008787199854850769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,8192,0.038424531618754074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,7168,0.04248960018157959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,7168,0.035658665498097736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,7168,0.02568853298823039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,6144,0.03740799824396769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,6144,0.03253333369890849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,6144,0.022997333606084188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,5120,0.031461334228515624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,5120,0.029770666360855104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,4096,0.025846399863560993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,4096,0.026497066020965576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,768,0.006885333359241486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,3584,0.02302186687787374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,3584,0.025205334027608235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,10240,512,0.005660800139109293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,3584,0.015564800302187601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,3072,0.02346453269322713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,3072,0.013737600048383078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,2560,0.01747200091679891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,2560,0.022322134176890055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,2560,0.012857600053151449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,2048,0.01443839967250824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,2048,0.020564266045888267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,3072,0.02040426731109619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,65536,0.18517333666483562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,1536,0.02022506594657898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,16384,0.05931413173675537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,1024,0.008678399523099263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,1024,0.01730560064315796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,8192,0.0286901334921519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,10240,0.03431146542231242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,768,0.00734506646792094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,768,0.01528320014476776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,768,0.006089599927266439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,512,0.005895466605822245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,512,0.005491200089454651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,512,0.015385599931081137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,256,0.0036938667297363283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,256,0.014846932888031007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,256,0.005291733145713806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,128,0.0031690667072931922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,5120,0.02016426722208659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,1536,0.011637333035469054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,4096,0.017112533251444496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,64,0.0029887999097506206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,8192,32,0.0030026666820049284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,64,0.014872533082962037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,32,0.014794666568438211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,65536,0.18483413060506185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,65536,0.3260021209716797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,2048,0.01099626620610555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,51200,0.25738347371419273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,51200,0.14897386233011883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,8192,128,0.014745600024859109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,51200,0.12993919849395752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,16384,0.05794346729914347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,16384,0.0950762669245402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,12288,0.062483199437459315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,12288,0.04580373366673787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,12288,0.036354132493336994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,10240,0.055414398511250816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,10240,0.041308800379435226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,10240,0.03098133405049642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,8192,0.0453439990679423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,8192,0.03567786614100139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,7168,0.03790613412857056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,7168,0.03315733273824056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,1536,0.009825066725413004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,6144,0.0334879994392395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,6144,0.030873600641886396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,6144,0.02101973295211792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,5120,0.028427733977635698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,1024,0.007784533500671387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,5120,0.028270934025446576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,5120,0.018181333939234413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,4096,0.02322773337364197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,4096,0.0252074658870697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,3584,0.020940800507863365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,3584,0.024605866273244223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,8192,128,0.004896000027656555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,3072,0.018569600582122803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,3072,0.02258560061454773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,2560,0.015974400440851848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,2560,0.021590399742126464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,2560,0.011803733309110005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,2048,0.01333440045515696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,2048,0.02014933427174886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,4096,0.015681067109107973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,65536,0.1643701394399007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,1536,0.0107914666334788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,1536,0.019692800442377725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,1024,0.008150400221347808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,1024,0.01604159971078237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,16384,0.051666132609049474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,768,0.006969599922498067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,768,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,768,0.0060693333546320595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,512,0.004013866682847341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,512,0.015147733688354491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,512,0.005594666798909505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,256,0.0034506666163603462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,8192,0.02583786646525065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,256,0.014915200074513755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,128,0.0031146667897701263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,256,0.0051466668645540874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,128,0.004964266717433929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,128,0.014842666188875833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,64,0.0029472000896930695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,7168,0.02337813377380371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,7168,32,0.002906666696071625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,64,0.014725333452224732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,7168,32,0.014672000209490457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,65536,0.15845012664794922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,65536,0.2814069430033366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,51200,0.12765759627024334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,51200,0.22258987426757812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,3584,0.014259200294812521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,16384,0.08143893082936605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,51200,0.11501226425170899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,3072,0.012761599818865457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,16384,0.052593068281809485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,12288,0.042768001556396484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,12288,0.05806506474812826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,10240,0.04841493368148804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,10240,0.038492798805236816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,10240,0.027803732951482134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,8192,0.03951893250147502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,8192,0.03373973369598389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,8192,0.023150932788848878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,7168,0.034406399726867674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,7168,0.031293867031733195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,7168,0.021254400412241616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,6144,0.0304202675819397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,6144,0.028769065936406452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,2048,0.010358400146166484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,5120,0.025309866666793822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,5120,0.026779733101526898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,5120,0.016458666324615477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,4096,0.02176213264465332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,4096,0.024132267634073893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,3584,0.018811732530593872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,3584,0.022832000255584718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,1536,0.009243733684221904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,7168,1024,0.006729599833488464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,3072,0.017010132471720375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,3072,0.022123734156290688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,65536,0.1438528060913086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,2560,0.016477866967519125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,2560,0.02066133419672648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,2560,0.010987733801205952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,2048,0.012531200051307678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,2048,0.01945599913597107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,2048,0.009672533472379048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,1536,0.010038399696350097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,1536,0.018203733364741008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,12288,0.0324565331141154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,1024,0.007565866907437642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,1024,0.015148799618085226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,1024,0.00597866674264272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,768,0.006291200220584869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,768,0.015387733777364096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,768,0.005530666808287303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,512,0.003849600007136663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,512,0.01514240006605784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,512,0.0053258667389551794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,256,0.0033088001112143196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,256,0.01479680041472117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,6144,0.01879040002822876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,4096,0.014006400108337402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,256,0.004974933465321859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,128,0.003067733347415924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,128,0.004849066833655039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,64,0.002922666569550832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,128,0.014544000228246054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,6144,32,0.002918400118748347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,64,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,6144,32,0.014636799693107605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,65536,0.13896533648173015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,51200,0.19060266812642415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,65536,0.2410890738169352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,16384,0.06324160099029541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,3584,0.013089066743850708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,51200,0.10771733125050861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,16384,0.0406826655069987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,3072,0.011717333396275838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,16384,0.04528319835662842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,51200,0.09926506678263346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,16384,0.0360586682955424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,12288,0.04877226750055949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,12288,0.03758399883906047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,8192,0.03362026611963908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,10240,0.02469546596209208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,10240,0.03403733174006145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,7168,0.02818346619606018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,8192,0.030466133356094362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,7168,0.028454399108886717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,7168,0.018905599912007652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,6144,0.02638079921404521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,5120,0.02146240075429281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,5120,0.02435520092646281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,5120,0.014846932888031007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,4096,0.01771199901898702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,4096,0.022636799017588298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,10240,0.04160746733347575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,3584,0.015849600235621132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,6144,0.024920533100763954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,6144,1536,0.008762666583061218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,3072,0.014030933380126953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,3072,0.020794665813446044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,65536,0.12447679837544759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,2560,0.013753599921862283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,3584,0.021975467602411904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,2560,0.010145066181818645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,2560,0.019497599204381308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,12288,0.02881173292795817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,2048,0.010610133409500122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,2048,0.01872106591860453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,2048,0.009020800391832989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,1536,0.007778133451938629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,1536,0.016898133357365928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,8192,0.020682666699091592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,1024,0.006741333504517872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,1024,0.006504533191521962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,768,0.005377066632111868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,768,0.005783466498057047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,768,0.015150933464368185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,512,0.0036298667391141256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,512,0.014962133765220643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,6144,0.01680213411649068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,256,0.0033173332611719764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,256,0.014916266997655234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,128,0.0030773334205150605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,128,0.014684800306955972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,128,0.004802133142948151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,64,0.0028064000109831494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,4096,0.012853333353996277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,32,0.002807466685771942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,64,0.014728533228238425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,32,0.014598400394121806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,65536,0.1856885274251302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,65536,0.11005120277404785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,51200,0.14678293863932293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,5120,1536,0.008705066641171773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,5120,1024,0.014993066589037577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,3584,0.012133333086967468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,16384,0.05013546546300253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,51200,0.08543039957682291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,16384,0.039572266737620036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,3072,0.010969600081443787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,12288,0.03967466751734416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,12288,0.03378239870071411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,10240,0.033429332574208576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,10240,0.031185066699981688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,51200,0.08953173160552978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,8192,0.027815467119216918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,8192,0.02741439938545227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,12288,0.025193599859873454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,256,0.0051594664653142296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,7168,0.025093332926432295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,5120,512,0.005435733497142792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,7168,0.016899200280507405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,7168,0.02609813412030538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,6144,0.022356265783309938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,6144,0.015003732840220132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,5120,0.01936639944712321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,6144,0.024296534061431885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,4096,0.01597653329372406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,5120,0.022674133380254112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,4096,0.011707733074824016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,4096,0.02102933327356974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,3584,0.014485333363215128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,3584,0.020487467447916664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,3584,0.01129813293615977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,3072,0.011910399794578553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,3072,0.019591466585795084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,3072,0.010276266932487487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,2560,0.010473600029945374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,2560,0.019058134158452353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,2048,0.009000533819198608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,2048,0.01790293256441752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,2048,0.00830506682395935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,1536,0.007640533149242401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,1536,0.015599999825159708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,65536,0.10732053120930989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,1024,0.006550399959087372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,1024,0.015406933426856995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,768,0.0039434666434923805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,768,0.015242666999499003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,16384,0.03092693289120992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,512,0.0035690667728583017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,512,0.014872533082962037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,10240,0.021925334135691324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,256,0.0032543999453385672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,256,0.014729600151379904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,8192,0.018024533987045288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,128,0.002977066735426585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,128,0.0145578662554423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,5120,0.013384532928466798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,64,0.0028405333558718365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,64,0.014805333813031516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,4096,32,0.0029418667157491045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,4096,32,0.014680533607800802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,2560,0.009711999694506328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,65536,0.16976745923360187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,65536,0.10292800267537434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,51200,0.13434987068176268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,1536,0.007299200197060903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,51200,0.08499626318613687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,16384,0.04841599861780803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,1024,0.005938133100668589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,16384,0.03680213292439778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,16384,0.030297599236170453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,12288,0.03476693232854207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,12288,0.031803733110427855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,12288,0.024050132433573405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,10240,0.030041599273681642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,768,0.005686399837334951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,10240,0.028919466336568195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,10240,0.021095466613769532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,8192,0.024860799312591553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,8192,0.025868799289067584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,7168,0.022592000166575112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,7168,0.02458239992459615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,6144,0.020228266716003418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,512,0.005222400029500326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,6144,0.0232586661974589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,5120,0.017384533087412515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,5120,0.02215893268585205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,256,0.005031466484069824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,4096,0.014568533500035605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,4096,0.020512000719706217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,4096,128,0.004704000055789947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,3584,0.013198933005332947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,3584,0.019873066743214925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,65536,0.10365227063496907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,3072,0.012075733145078022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,3072,0.019341866175333657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,51200,0.08249066670735677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,2560,0.0095551997423172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,2560,0.01839253306388855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,8192,0.01786880095799764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,2048,0.008447999755541485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,7168,0.016328533490498863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,6144,0.013963733116785684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,2048,0.017350399494171144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,1536,0.0072405333320299785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,5120,0.01309333344300588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,1536,0.015707733233769734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,1024,0.004836266736189524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,1024,0.015345066785812378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,4096,0.011429333686828613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,768,0.003980800012747447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,768,0.015155200163523355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,768,0.005756799876689911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,512,0.0036085332433382668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,512,0.01495253344376882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,3584,0.011126400033632914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,256,0.0031658666829268134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,256,0.014556800325711569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,256,0.004984533290068308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,128,0.0029077333708604175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,128,0.01472106675306956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,128,0.004773333172003428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,64,0.002834133307139079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,64,0.014525866508483887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3584,32,0.0028768000503381092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3584,32,0.014533332983652749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,65536,0.14457066853841144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,3072,0.010005333026250203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,65536,0.08953173160552978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,65536,0.09541119734446207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,51200,0.11468799908955891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,2560,0.009337600072224934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,51200,0.07247359752655029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,16384,0.040040532747904464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,16384,0.03548479874928792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,16384,0.028066132465998334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,12288,0.032203733921051025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,12288,0.030187733968098956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,12288,0.022554665803909302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,10240,0.02704319953918457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,10240,0.027342933416366576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,8192,0.023180800676345825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,10240,0.019713066021601357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,2048,0.007669333120187123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,8192,0.024971733490626015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,7168,0.020909865697224937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,7168,0.023909332354863484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,6144,0.01862186590830485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,1536,0.006866133213043213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,6144,0.022317866484324135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,5120,0.015660799543062844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,5120,0.020770132541656494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,5120,0.012435199817021687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,4096,0.013871999581654868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,4096,0.020218666394551596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,3584,0.01234879990418752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,1024,0.006182399888833364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,3584,0.019223467508951823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,3072,0.011191466450691223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,3072,0.018678400913874307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,3072,0.009628799557685853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,2560,0.008969600001970928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,2560,0.0173632005850474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3584,512,0.005335466563701629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,2048,0.007941333452860515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,2048,0.015920000274976094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,51200,0.07596373558044434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,1536,0.006718933085600535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,1536,0.01562773287296295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,1536,0.006656000018119812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,1024,0.004465066889921824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,1024,0.015319466590881348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,8192,0.01667840083440145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,768,0.003988266736268997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,7168,0.01546986699104309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,768,0.014940800269444785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,6144,0.014056533575057983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,512,0.0036202666660149893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,512,0.015108266472816467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,256,0.0031648000081380212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,4096,0.010577066739400228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,256,0.01474453310171763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,128,0.0029781334102153777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,256,0.004984533290068308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,128,0.014616533120473226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,128,0.004862933357556661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,64,0.002846933404604594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,3072,32,0.0028704000016053517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,64,0.014398933450380961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,3072,32,0.014552533626556396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,65536,0.12128106753031413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,65536,0.07794559796651204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,2560,0.008686932921409606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,51200,0.09638293584187826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,2048,0.0075573335091273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,51200,0.07422293027242025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,16384,0.03380053440729777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,16384,0.027507199843724565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,16384,0.031123199065526325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,12288,0.026661332448323565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,12288,0.02720639904340108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,12288,0.02177813251813253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,10240,0.02328746716181437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,10240,0.025231999158859254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,10240,0.019349332650502524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,512,0.0052597333987553915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,8192,0.019383466243743895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,8192,0.023002666234970093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,7168,0.017458132902781167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,7168,0.022011733055114745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,3584,0.010702932874361675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,6144,0.015522133310635886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,6144,0.02100373307863871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,6144,0.01378773351510366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,5120,0.013615999619166055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,5120,0.020436267058054604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,51200,0.06230400005976359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,4096,0.011770666639010111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,4096,0.019244800011316933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,1024,0.005967999994754791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,3584,0.01099733312924703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,3584,0.01872319976488749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,3072,768,0.005565866827964783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,3072,0.010043733318646749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,3072,0.017853866020838417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,3072,0.009196799993515015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,2560,0.00843519965807597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,2560,0.01612160007158915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,2560,0.008404266834259034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,2048,0.00722453345855077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,2048,0.01602453291416168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,2048,0.007180800040562947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,1536,0.005781333148479462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,1536,0.015717333555221556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,65536,0.08628160158793131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,1024,0.0042250668009122215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,1024,0.01511679987112681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,3584,0.010403199990590414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,768,0.0039488000174363455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,768,0.01515733301639557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,512,0.0035103999078273775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,8192,0.01641386648019155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,512,0.014986667037010192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,512,0.005230933427810669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,256,0.0031328000128269195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,256,0.005205333232879639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,256,0.014791466792424521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,128,0.002951466788848241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,128,0.004759466648101807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,128,0.014390400052070618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,64,0.002811733384927114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,7168,0.01520639955997467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2560,32,0.002880000074704488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,64,0.014607999722162882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2560,32,0.014476799964904785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,65536,0.09630506833394369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,65536,0.06389973163604737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,65536,0.08590826988220215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,51200,0.07574293613433838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,16384,0.027581866582234698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,16384,0.027619200944900512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,5120,0.012410666545232136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,4096,0.0108842670917511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,12288,0.022115200757980347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,12288,0.024171733856201173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,12288,0.020413867632548013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,10240,0.01932586630185445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,10240,0.02302293380101522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,10240,0.018143999576568603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,8192,0.01616320013999939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,8192,0.02137920061747233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,8192,0.01548799971739451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,7168,0.01453013320763906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,7168,0.020936532815297445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,7168,0.014268799622853597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,6144,0.013031466801961263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,6144,0.020003199577331543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,5120,0.011699199676513672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,5120,0.019800533850987755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,51200,0.054093865553538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,4096,0.010173867146174114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,4096,0.018632533152898152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,4096,0.01018453339735667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,3584,0.009690666198730468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,3584,0.017709867159525553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,1536,0.006628266473611195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,3584,0.009581866860389709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,3072,0.009006933371225993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,3072,0.008610133330027263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,3072,0.01696746746699015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,2560,0.007584000130494435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,2560,0.016566399733225504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,2560,0.00795839975277583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,2048,0.006678399940331777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,1024,0.005717333157857259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,1536,0.004743466774622599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,1536,0.015507200360298156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,1536,0.006546133259932201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,1024,0.004112000018358231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,1024,0.015321600437164306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2560,768,0.005678933362166087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,51200,0.06795413494110107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,768,0.0037418665985266366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,768,0.014919466773668923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,16384,0.025383466482162477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,768,0.0054293334484100345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,2048,0.01602026621500651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,512,0.0034517332911491393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,256,0.003102933367093404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,512,0.014617600043614707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,256,0.014416000247001648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,256,0.004915200173854828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,128,0.0027722666660944624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,128,0.014622933665911355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,128,0.004677333434422811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,64,0.0026634665826956432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,2048,32,0.0026869334280490874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,64,0.014324266711870828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,2048,32,0.014457600315411887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,65536,0.07455039819081624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,6144,0.013127467036247254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,65536,0.05369493166605631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,5120,0.011641599734624227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,65536,0.08374293645222983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,51200,0.05965973138809204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,16384,0.022379734118779502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,51200,0.04507946570714315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,51200,0.06674559911092123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,16384,0.024519467353820802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,12288,0.02211306691169739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,12288,0.02001919945081075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,10240,0.015478400389353433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,10240,0.021388800938924153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,8192,0.013310933113098144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,8192,0.020179200172424316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,2048,0.007068799932797749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,7168,0.01234773298104604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,7168,0.0197653333346049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,7168,0.013939199844996133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,6144,0.011267200112342834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,6144,0.019511467218399046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,12288,0.018038400014241538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,1024,0.005730133255322774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,6144,0.012714667121569314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,5120,0.010227200388908387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,4096,0.009179733196894328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,5120,0.018439465761184694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,4096,0.017026132345199584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,3584,0.008669867118199667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,3584,0.017204266786575318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,3584,0.008926933010419209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,3072,0.008056533336639405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,3072,0.0167413334051768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,3072,0.008136533200740814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,2560,0.007110400001207988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,2560,0.01702293356259664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,2048,0.005499733487764994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,2048,0.01600213348865509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,2048,0.006922666728496551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,1536,0.004769066472848257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,1536,0.01569386621316274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,1536,0.006471466521422069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,1024,0.004110933343569437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,1024,0.01535146633783976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,2048,512,0.005233066777388254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,768,0.0037834666669368743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,768,0.015095466375350952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,768,0.00553173323472341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,512,0.0033610666791598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,512,0.015009066462516785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,16384,0.024809600909550984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,256,0.0030048000315825146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,10240,0.017573332786560057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,256,0.014371200402577718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,128,0.0028757333755493166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,128,0.014432000120480857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,8192,0.015203199783960977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,64,0.002705066651105881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,64,0.014338133732477823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1536,32,0.0027242665489514667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1536,32,0.0143477330605189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,65536,0.0502623995145162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,65536,0.04172906478246053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,65536,0.0840885321299235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,51200,0.0405898650487264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,51200,0.03593493302663167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,5120,0.011367467045783997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,16384,0.01609813372294108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,4096,0.009583999713261921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,16384,0.021758933862050377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,16384,0.02478826642036438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,12288,0.013431466619173684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,12288,0.01997119983037313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,12288,0.020096000035603842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,10240,0.012110933661460876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,10240,0.019621332486470543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,8192,0.010446932911872864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,2560,0.00772159993648529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,8192,0.0191648006439209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,7168,0.009873066345850627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,7168,0.017977599302927652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,1024,0.005763199925422668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,7168,0.013418666521708169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,6144,0.00888320008913676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,6144,0.017864533265431724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,6144,0.01188053290049235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,5120,0.008567466338475545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,5120,0.01798293391863505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,512,0.00513919989267985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,4096,0.0075882668296496075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,4096,0.017537067333857216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,256,0.004987733562787374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,3584,0.007181866466999054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,3584,0.016934400796890257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,3584,0.008897067109743754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,3072,0.006472533444563548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,3072,0.016569599509239197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,51200,0.06694186528523763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,2560,0.0060928001999855045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,2560,0.016587733229001363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,2560,0.007712000111738841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,2048,0.005376000205675761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,10240,0.01763199965159098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,2048,0.01609386702378591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,1536,0.0047082667549451195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,8192,0.014858667055765787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,1536,0.015770666797955833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,1024,0.004049066702524821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,1024,0.015273599823315939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,768,0.003755733370780945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,768,0.014917332927385965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,5120,0.01067200005054474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,512,0.003389866650104523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,512,0.014818132917086283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1536,128,0.004762666424115499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,256,0.003014400104681651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,4096,0.009426132837931315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,256,0.014412800470987955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,128,0.002855466554562251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,128,0.014306132992108664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,1536,0.006494933366775512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,64,0.0026890667776266735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,3072,0.008062933385372163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,1024,32,0.0027872001131375628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,64,0.014644267161687216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,1024,32,0.014389333128929139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,65536,0.03968426783879598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,65536,0.036030932267506914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,51200,0.03237119913101196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,51200,0.03179946740468343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,51200,0.06636586586634317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,16384,0.013563733299573263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,16384,0.020386133591334024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,16384,0.024765866994857787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,12288,0.011126400033632914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,12288,0.019975467522939046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,2048,0.006796800096829732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,10240,0.010505599776903789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,10240,0.01835093299547831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,1024,0.005783466498057047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,8192,0.009360000491142273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,8192,0.01751040021578471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,8192,0.014476799964904785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,7168,0.008574933807055155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,7168,0.01808746655782064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,7168,0.01313920021057129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,6144,0.008260266482830047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,768,0.005525333185990652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,6144,0.017884800831476845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,5120,0.007415466507275899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,6144,0.011708799997965496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,5120,0.018297600746154784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,512,0.005153066913286845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,4096,0.0063509335120519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,5120,0.010537599523862202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,3584,0.006679466863473256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,4096,0.009299199779828389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,4096,0.017898666858673095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,3584,0.00900266667207082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,3584,0.017043199141820273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,3072,0.008108800152937572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,3072,0.01676586667696635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,2560,0.00602346658706665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,2560,0.016155733664830526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,128,0.004701866706212362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,2048,0.0054624001185099285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,2048,0.006821333368619282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,2048,0.01576959987481435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,1536,0.0047189335028330484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,1536,0.006555733581384023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,1536,0.01542080044746399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,1024,0.004098133246103922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,1024,0.015007999539375306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,1024,0.0057322666049003605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,768,0.003705599904060364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,768,0.015130666891733804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,65536,0.08363839785257975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,512,0.003389866650104523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,512,0.014818132917086283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,256,0.0031082667410373688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,256,0.014615466197331747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,12288,0.01959679921468099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,256,0.004964266717433929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,128,0.002855466554562251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,3072,0.006128000219662985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,128,0.014418133099873862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,64,0.0027978666126728057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,64,0.014506666858990987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,768,32,0.002771199991305669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,768,32,0.014564266800880432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,65536,0.028252800305684406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,65536,0.029360000292460126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,65536,0.08360426425933838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,10240,0.01732800006866455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,51200,0.023362133900324503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,51200,0.026121600468953447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,51200,0.06620479822158813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,16384,0.010618666807810467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,16384,0.018913066387176512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,12288,0.009065600236256917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,12288,0.017385600010553996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,1024,256,0.00486826648314794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,10240,0.008589866757392883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,10240,0.017949867248535156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,10240,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,8192,0.007378133138020833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,8192,0.017822933197021485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,8192,0.014199466506640116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,7168,0.006704000135262807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,2560,0.0077237332860628765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,7168,0.018071466684341432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,7168,0.013006933530171714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,6144,0.006381866832574208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,6144,0.011725866794586181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,6144,0.017986132701237997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,5120,0.006433066725730896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,5120,0.018147200345993042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,16384,0.024519467353820802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,4096,0.005895466605822245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,4096,0.009290666381518046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,4096,0.017566933234532674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,3584,0.0063391998410224915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,768,0.00544213354587555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,3584,0.017110399405161538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,3072,0.006021333237489065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,3072,0.008057599763075511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,2560,0.006145066519578298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,512,0.005126399795214335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,2560,0.01624853312969208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,2048,0.005379199981689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,2048,0.015893333156903586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,768,128,0.0046741331617037455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,1536,0.004762666424115499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,1536,0.015737600127855935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,1536,0.006390400230884552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,1024,0.004156800111134847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,1024,0.015179733435312906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,1024,0.005757866799831391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,768,0.0038143999874591826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,768,0.014998400211334228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,512,0.0035018667578697203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,512,0.01488746702671051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,3072,0.016838399569193523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,256,0.0031637333333492277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,256,0.01455573340257009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,12288,0.01939093271891276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,128,0.002850133428970973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,128,0.014386133352915446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,128,0.004771199822425842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,64,0.002746666719516118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,64,0.014412800470987955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,512,32,0.00276799996693929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,512,32,0.014320000012715658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,65536,0.01764799952507019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,65536,0.02379306753476461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,5120,0.010554666320482891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,51200,0.01378666659196218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,51200,0.022005333503087362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,3584,0.00897920032342275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,51200,0.06674453417460123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,16384,0.017697066068649292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,2560,0.007603199779987335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,12288,0.006454400221506755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,2048,0.006775466601053874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,12288,0.01895573337872823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,10240,0.006325333317120869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,10240,0.017947733402252197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,768,0.005426133175690969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,8192,0.006098133325576782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,512,0.00513919989267985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,16384,0.007758933305740357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,8192,0.017523199319839478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,7168,0.006095999975999197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,8192,0.014176000157992044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,6144,0.00594346672296524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,7168,0.017892267306645712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,6144,0.01744106610616048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,512,256,0.004897066454092661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,5120,0.0063381334145863845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,5120,0.010673066973686219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,5120,0.017854932943979898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,4096,0.005883733431498209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,4096,0.01739306648572286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,12288,0.01767680048942566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,3584,0.006340266764163971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,3584,0.016960000991821288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,65536,0.0832266648610433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,3072,0.005992533266544342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,3072,0.016396799683570863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,16384,0.02414720058441162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,2560,0.006055466830730438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,2560,0.016360533237457276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,2560,0.007708799839019775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,2048,0.00535999983549118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,2048,0.016114133596420287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,2048,0.006799999872843425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,10240,0.01646506687005361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,1536,0.0046517332394917805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,1536,0.01543786625067393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,1024,0.003996799886226654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,1024,0.015277866522471109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,1024,0.005682133138179779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,768,0.003685333331425985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,768,0.014999467134475707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,6144,0.011680000027020772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,7168,0.01302186648050944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,512,0.0033919999996821085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,512,0.014786133170127868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,256,0.0030271999537944795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,256,0.014365866780281067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,4096,0.00920853316783905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,128,0.0028351999819278715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,128,0.014391466975212097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,64,0.002679466704527537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,64,0.014257066448529563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,256,32,0.002656000107526779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,256,32,0.014416000247001648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,65536,0.011762133240699768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,65536,0.02079040010770162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,3072,0.00801813354094823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,51200,0.010236799716949463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,51200,0.018875734011332194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,3584,0.008730666836102803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,16384,0.00600853314002355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,16384,0.01785599986712138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,16384,0.023973333835601806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,12288,0.005825066566467285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,1536,0.006489600241184235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,12288,0.017382399241129557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,10240,0.006180266539255777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,10240,0.01790293256441752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,768,0.005382399757703145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,8192,0.005914666752020518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,8192,0.017799466848373413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,8192,0.014019200205802917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,7168,0.005853866537412008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,7168,0.017564799388249716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,512,0.005050666630268097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,6144,0.005773866673310598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,6144,0.017654399077097573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,256,0.004887466629346212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,5120,0.006106666723887126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,5120,0.018382932742436728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,256,128,0.004757333298524221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,4096,0.005746133128801982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,4096,0.017217065890630087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,12288,0.01904426614443461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,3584,0.0061258668700853985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,3584,0.016733866930007935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,65536,0.08297386964162191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,3072,0.005880533158779145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,51200,0.06556693315505982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,3072,0.01635199983914693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,2560,0.005717333157857259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,2560,0.016032000382741295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,10240,0.016611199577649435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,2048,0.0052159999807675685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,2048,0.015793066223462424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,7168,0.012937600413958231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,1536,0.004477866490681966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,1536,0.015422933300336204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,6144,0.011548800269762675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,1024,0.00395413339138031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,1024,0.014929067095120749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,5120,0.010364799698193868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,768,0.0035978667438030243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,768,0.014857600132624308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,4096,0.009204266468683879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,512,0.0032960000137488045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,512,0.014932266871134438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,512,0.005046399931112925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,256,0.0030048000315825146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,256,0.01474666694800059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,3072,0.008026666442553202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,128,0.002882133424282074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,128,0.01432960033416748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,128,0.004746666550636292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,64,0.002674133330583572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,2560,0.007482666770617168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,64,0.0144405335187912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,128,32,0.0026357332865397137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,128,32,0.01420906682809194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,65536,0.008941866954167684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,65536,0.01936639944712321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,51200,0.007326933244864146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,51200,0.01909439961115519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,16384,0.006000000238418579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,16384,0.017989333470662436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,12288,0.005998933315277099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,12288,0.01740480065345764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,10240,0.006073600053787232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,10240,0.018227199713389076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,8192,0.005890133480230967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,8192,0.017517866690953572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,3584,0.00879146655400594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,7168,0.005824000140031179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,6144,0.0056981335083643595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,7168,0.017683200041453042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,5120,0.005974400043487549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,5120,0.018231467405954997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,4096,0.005619200070699056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,4096,0.017036799589792886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,3584,0.006040533383687338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,3584,0.016730666160583496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,3072,0.005745066702365876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,3072,0.01641386648019155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,2560,0.005688533186912537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,2048,0.006871466835339864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,2048,0.005037866532802582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,2560,0.016055466731389363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,1536,0.004481066763401031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,2048,0.01591679950555166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,1024,0.003773866593837738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,1536,0.015468800067901611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,1536,0.0063296000162760425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,768,0.003521066655715307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,1024,0.015250133474667868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,512,0.003278933217128118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,768,0.014849066734313965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,256,0.0029450667401154833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,512,0.014685866236686707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,128,0.002775466690460841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,256,0.014461867014567056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,64,0.002629333237806956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,128,0.014351999759674073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,64,32,0.002629333237806956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,64,0.014358400305112203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,65536,0.008943999807039898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,32,0.014454399545987448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,51200,0.007892266909281413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,65536,0.019631999731063842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,51200,0.01880533297856649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,16384,0.005825066566467285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,1024,0.005701333284378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,12288,0.0058112000425656635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,16384,0.01734613378842672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,10240,0.005998933315277099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,8192,0.0058890665570894875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,10240,0.01770133376121521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,7168,0.005737600227197012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,8192,0.017259732882181803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,6144,0.005584000051021576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,64,6144,0.01749546726544698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,5120,0.0059232001503308615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,6144,0.017468800147374473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,4096,0.005527466535568237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,5120,0.018122667074203493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,3584,0.006002133091290792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,4096,0.01684373418490092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,3072,0.005685333410898844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,3584,0.016583466529846193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,2560,0.00561599979797999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,3072,0.016083199779192606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,2560,0.016059733430544534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,1536,0.00450133333603541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,2048,0.01562879979610443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,1024,0.0038272000849246977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,256,0.0047423998514811196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,1536,0.015525333086649575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,768,0.003585066646337509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,1024,0.014964266618092855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,512,0.0032853332658608755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,768,0.014922666549682616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,256,0.0029397333661715193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,512,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,128,0.0027424000203609467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,256,0.014422399799029031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,64,0.0025909334421157838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,128,0.01421119968096415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,32,0.00264533335963885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,64,0.014433067043622336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,32,0.014327466487884521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,12288,0.017369600137074788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,12,32,7168,0.017842133839925133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,12,128,768,0.005410133302211762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,12,32,2048,0.005018666883309682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,16384,0.748195203145345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,16384,0.39179201126098634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,12288,0.2982069333394369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,12288,0.5719455718994141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,12288,0.2732725461324056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,10240,0.24975892702738442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,10240,0.4770911852518718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,51200,1.1820149739583334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,8192,0.20410879453023276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,8192,0.37805439631144205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,8192,0.21147626241048179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,7168,0.18018026351928712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,51200,2.3390772501627604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,7168,0.3299189249674479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,6144,0.28538452784220375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,6144,0.1377514680226644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,6144,0.18225919405619304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,5120,0.23742186228434242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,5120,0.13391253153483074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,4096,0.19253759384155272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,4096,0.10962879657745361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,3584,0.16778987248738605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,3584,0.09847466945648194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,16384,0.3570058822631836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,3072,0.14495360056559245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,3072,0.08715519905090333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,3072,0.07242879867553711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,10240,0.2252789338429769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,51200,1.093742879231771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,2560,0.121562663714091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,2560,0.07527573108673095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,2560,0.07027093569437662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,2048,0.06307200193405152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,1536,0.07495146592458089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,2048,0.11371733347574871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,1536,0.052218667666117345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,1536,0.04001706838607788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,1024,0.05917653242746988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,1024,0.039956267674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,1024,0.028935466210047407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,768,0.04592853387196859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,768,0.03438613414764404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,768,0.023037866751352946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,512,0.027490133047103883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,256,0.015479466319084168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,256,0.021115734179814657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,256,0.01204266647497813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,128,0.009176533420880635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,128,0.01790720025698344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,128,0.009324799974759419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,64,0.006386133531729381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,64,0.01623040040334066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,65536,32,0.004514133433500925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,32,0.016158933440844216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,5120,0.11643413702646892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,4096,0.0944426695505778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,65536,512,0.02764906684557597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,7168,0.15932052930196125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,3584,0.08369599978129069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,2048,0.05079040129979452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,65536,1.2043008168538412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,51200,0.9513109207153321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,65536,512,0.017196800311406454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,16384,0.6094581604003906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,65536,2.355164845784505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,51200,1.8576170603434246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,16384,0.34548053741455076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,12288,0.4665205319722493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,12288,0.23904959360758463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,10240,0.20064746538798012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,8192,0.3066922823588053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,12288,0.2594666639963786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,10240,0.4495242754618327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,8192,0.16910613377888997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,8192,0.15997653007507323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,7168,0.3016256014506022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,7168,0.14592746098836262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,6144,0.23495680491129556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,6144,0.11532373428344726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,6144,0.14468266169230143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,5120,0.19570560455322267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,5120,0.10831999778747559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,4096,0.09072213172912598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,4096,0.15252587000528972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,4096,0.07930986881256104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,3584,0.1359989325205485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,3584,0.08081706364949545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,3072,0.11542613506317138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,3072,0.0724234660466512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,65536,1.1601898193359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,51200,0.9053418477376303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,2560,0.1102357308069865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,2560,0.06874026457468668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,16384,0.3118101437886556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,2560,0.05881066719690958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,2048,0.08889173666636149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,2048,0.05775359869003296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,1536,0.062244268258412686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,2048,0.04793813228607178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,1536,0.04360959927241008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,1536,0.03416533470153808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,10240,0.18699199358622234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,1024,0.03383253415425618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,768,0.03561600049336751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,768,0.028945066531499225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,768,0.020261333386103312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,512,0.02264960010846456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,512,0.023788799842198692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,7168,0.1332810640335083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,256,0.013582932949066161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,256,0.01950506567955017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,1024,0.04067306518554688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,128,0.007717333237330119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,128,0.01532906691233317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,5120,0.09736106395721436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,64,0.00432640016078949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,64,0.016082132856051125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,51200,32,0.004193066557248434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,51200,32,0.016151466965675355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,3584,0.07040639718373617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,3072,0.06141440073649088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,65536,0.375272528330485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,65536,0.7419989267985027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,65536,0.3556480089823405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,1024,0.024781866868336996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,16384,0.18341867129007977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,16384,0.10689386526743572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,51200,0.3010869344075521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,512,0.015398400028546652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,51200,0.5823914845784505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,12288,0.13899413744608563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,51200,0.27932907740275065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,12288,0.07247893015543619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,12288,0.0966048002243042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,10240,0.07227946917215983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,8192,0.061546667416890466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,8192,0.09490559895833334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,10240,0.1365909258524577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,8192,0.049941333134969075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,7168,0.056034131844838464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,6144,0.07200640042622884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,6144,0.05025920073191324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,6144,0.039153067270914714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,256,0.010858666896820069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,5120,0.06031573216120402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,5120,0.04431893428166707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,4096,0.04931626717249553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,4096,0.028060799837112425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,3584,0.04364266792933146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,3584,0.035589333375295004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,3584,0.02546773354212443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,3072,0.03785173495610555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,51200,128,0.007942399879296621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,3072,0.03271039923032125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,3072,0.022504534324010214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,2560,0.03215786616007487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,2560,0.029857067267100017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,2048,0.026338134209314985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,2048,0.026477867364883424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,7168,0.08239999612172445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,1536,0.020694400866826376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,1536,0.023348265886306764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,1536,0.013754666845003764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,1024,0.014634666840235391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,1024,0.020396800835927327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,4096,0.03855679829915364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,768,0.01165013313293457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,768,0.019141334295272826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,16384,0.09385493596394857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,512,0.009828266501426697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,512,0.01763199965159098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,10240,0.07113920052846273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,256,0.006489600241184235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,7168,0.04801493485768636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,256,0.015281066298484802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,256,0.00544213354587555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,128,0.0034944000343481696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,128,0.014979199568430582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,5120,0.03372373183568318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,64,0.015099733074506124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,32,0.003310933212439219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,16384,32,0.015001599987347921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,2560,0.019850667317708334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,2048,0.016702934106191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,65536,0.5553066889444987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,65536,0.2983423868815104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,65536,0.283297061920166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,16384,64,0.0032255999743938447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,1024,0.01050986647605896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,51200,0.43610668182373047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,16384,0.14489173889160156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,768,0.009410132964452107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,16384,0.08911680380503337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,12288,0.11122666994730632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,51200,0.22117652893066406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,51200,0.23718080520629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,16384,0.08743360042572021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,12288,0.0678933302561442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,10240,0.09302293459574382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,8192,0.05049386819203695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,10240,0.06047039826711019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,8192,0.08396906852722168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,7168,0.06464106639226277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,7168,0.04744426806767781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,6144,0.05526506503423055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,6144,0.042624000708262125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,6144,0.0318069338798523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,5120,0.047990401585896805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,5120,0.0381386677424113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,512,0.007812266548474629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,4096,0.03977173169453939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,4096,0.033522133032480875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,4096,0.02318293253580729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,3584,0.03498560190200806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,3584,0.03178559939066569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,16384,128,0.004923733572165171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,3584,0.021175465981165566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,3072,0.030364799499511718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,3072,0.029029333591461183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,2560,0.025867732365926106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,2560,0.02680533329645793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,12288,0.05797119935353597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,10240,0.05567040046056112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,2048,0.021197867393493653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,8192,0.040611199537913006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,2048,0.01404159963130951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,1536,0.01646933356920878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,7168,0.03623253504435221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,1024,0.01186133325099945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,1536,0.021569067239761354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,1024,0.009455999732017517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,1024,0.019500799973805747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,768,0.009755733609199523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,512,0.007415466507275899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,768,0.00819413314263026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,768,0.018077866236368815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,512,0.005826133489608765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,256,0.003833599885304769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,512,0.015561599532763162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,128,0.0034272000193595886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,256,0.005212800204753875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,256,0.015117866794268289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,64,0.003138133386770884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,128,0.004949333270390829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,128,0.014856533209482829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,12288,32,0.003124266614516576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,64,0.014945066968599954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,32,0.014813866217931113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,5120,0.02739306688308716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,65536,0.23933653831481932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,65536,0.25983467102050783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,51200,0.3769322713216146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,65536,0.4796714782714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,3072,0.018809600671132406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,16384,0.08578453063964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,12288,2048,0.023933867613474526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,51200,0.18309653600056966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,2560,0.016804265975952148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,16384,0.0640010674794515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,12288,0.06095573504765829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,12288,0.09517013231913249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,10240,0.07854080200195312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,10240,0.051261866092681886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,8192,0.06530346473058066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,8192,0.04434773524602254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,8192,0.03490240176518758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,7168,0.040977064768473306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,7168,0.05299093325932821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,6144,0.046510934829711914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,6144,0.03725013335545858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,6144,0.027918932835261027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,5120,0.03898239930470784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,5120,0.03345813353856404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,4096,0.03195199966430664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,4096,0.029710932572682695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,16384,0.12480746905008952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,51200,0.20249279340108237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,3584,0.0321834663550059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,3584,0.01867626706759135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,3072,0.02478613257408142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,3584,0.030612266063690184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,3072,0.01672533353169759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,3072,0.02534079949061076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,2560,0.021266132593154907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,2560,0.024010666211446128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,12288,1536,0.0116757333278656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,2048,0.017437867323557534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,2048,0.021988266706466676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,1536,0.013749333222707114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,1536,0.020281600952148437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,2048,0.012972799936930337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,12288,0.04962773323059082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,1024,0.010198400417963664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,10240,0.04126186768213908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,1024,0.018759467204411826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,768,0.008575999736785888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,768,0.016961065928141277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,512,0.006625066697597504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,7168,0.031137067079544067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,512,0.015516799688339234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,256,0.003566933423280716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,5120,0.024091732501983643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,256,0.015146666765213012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,128,0.0032490665713946023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,128,0.014914133151372275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,4096,0.020822399854660036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,2560,0.014728533228238425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,64,0.0030080000559488933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,1536,0.010771200060844421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,64,0.014945066968599954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,10240,32,0.014903466900189719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,10240,32,0.003011200080315272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,65536,0.19866132736206055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,65536,0.3751765251159668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,51200,0.29614826838175456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,1024,0.00865600009759267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,51200,0.16302933692932128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,768,0.0070709332823753355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,51200,0.14638932545979816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,16384,0.09228586355845134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,256,0.005300266544024149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,512,0.005693866809209188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,16384,0.06089599927266439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,12288,0.050115199883778896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,12288,0.04031573136647542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,10240,0.04404693444569906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,10240,0.03423253297805786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,8192,0.048121599356333415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,8192,0.038507731755574544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,8192,0.028849067290623982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,10240,128,0.004934399823347728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,7168,0.04256319999694824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,7168,0.035146665573120114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,6144,0.037512532869974774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,6144,0.03280959924062093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,5120,0.031649067004521685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,5120,0.029821866750717164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,4096,0.025851732492446898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,4096,0.026834134260813398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,12288,0.07073173522949219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,3584,0.02300693392753601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,10240,0.05951040188471476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,3584,0.024910932779312132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,3584,0.015602133671442666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,3072,0.023568000396092734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,65536,0.18590614000956218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,2560,0.019754666090011596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,2560,0.02230506738026937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,2560,0.012703999876976013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,2048,0.014444800217946372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,2048,0.020431999365488687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,2048,0.01128426690896352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,1536,0.01146986683209737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,1536,0.019138133525848387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,1536,0.009750399986902874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,1024,0.008557867010434468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,16384,0.05108480056126913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,1024,0.017683200041453042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,3072,0.020337067047754922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,768,0.00730453332265218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,768,0.015783466895421348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,512,0.005721599857012431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,512,0.015379200379053751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,7168,0.02600746750831604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,256,0.0036202666660149893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,256,0.015103999773661295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,6144,0.022980266809463502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,128,0.0031413334111372627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,128,0.01476693352063497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,128,0.00480320006608963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,4096,0.017518933614095053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,64,0.003052799900372823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,8192,32,0.0030591999491055804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,64,0.014994133512179056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,8192,32,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,65536,0.18371413548787435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,65536,0.3259765307108561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,51200,0.256768004099528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,3072,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,5120,0.02006933291753133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,51200,0.15051306088765462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,16384,0.09164266586303711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,16384,0.05744426647822062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,16384,0.04579306840896606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,12288,0.06293013493220011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,12288,0.04593706528345744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,12288,0.036098134517669675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,10240,0.05528853336970011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,10240,0.04083626667658488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,10240,0.031174399455388385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,8192,0.04520746469497681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,8192,0.03603839874267578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,7168,0.03798293272654216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,7168,0.03357653220494588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,1024,0.007830399771531422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,6144,0.03326080044110616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,768,0.006080000102519989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,6144,0.030800000826517744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,5120,0.028259199857711793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,512,0.005638400216897329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,5120,0.028350933392842607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,5120,0.018170666694641114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,4096,0.023176532983779908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,4096,0.02544959982236226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,3584,0.02096959948539734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,8192,256,0.005163733164469401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,3584,0.02410773237546285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,3584,0.014181333780288696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,3072,0.018542933464050292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,3072,0.02400853236516317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,3072,0.012681600451469422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,2560,0.0159850666920344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,2560,0.021569067239761354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,2560,0.011815466483434041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,2048,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,2048,0.020126932859420778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,65536,0.16399787267049154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,1536,0.012012799580891926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,1536,0.01893226703008016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,1536,0.00928000013033549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,1024,0.008151466647783916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,1024,0.015388799707094827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,51200,0.13013333479563396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,768,0.007527466615041096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,768,0.015572266777356467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,768,0.0056202664971351625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,512,0.004228266576925913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,512,0.015171200037002563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,8192,0.025914667050043742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,256,0.0033781332274278007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,256,0.014843733112017313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,7168,0.02328959902127584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,128,0.0031040000418821974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,6144,0.02093013326327006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,128,0.014806399742762247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,64,0.002924799919128418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,64,0.014812800288200378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,7168,32,0.0029002666473388673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,4096,0.015286399920781454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,7168,32,0.014808533589045205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,65536,0.15837225914001465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,65536,0.28150078455607097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,65536,0.14405867258707683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,51200,0.22230079968770347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,2048,0.010233599940935771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,16384,0.07545173168182373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,16384,0.051628800233205166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,51200,0.1274229367574056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,51200,0.11424427032470703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,12288,0.05824640194574991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,12288,0.032467200358708696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,10240,0.04833173354466756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,10240,0.027854933341344194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,10240,0.038624000549316403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,8192,0.0389631986618042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,8192,0.03441280126571655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,8192,0.02302079995473226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,7168,0.034891732533772785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,7168,0.03161279956499736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,6144,0.02990399996439616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,6144,0.029658667246500653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,1024,0.00661653329928716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,5120,0.025574400027592974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,5120,0.026792534192403156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,5120,0.01630400021870931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,4096,0.021580799420674642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,4096,0.024600533644358318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,512,0.005454933146635691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,3584,0.01935360034306844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,3584,0.02332693338394165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,12288,0.04320106506347656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,3072,0.016978132724761962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,3072,0.022146133581797282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,256,0.005176533261934916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,3072,0.011766399939854939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,2560,0.014843733112017313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,2560,0.020796799659729005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,2048,0.012483200430870056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,7168,128,0.004857600231965383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,2048,0.019453867276509603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,1536,0.01011199951171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,2048,0.009761066238085429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,1536,0.018304000298182167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,1536,0.008781866232554118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,1024,0.007546666761239369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,1024,0.015442132949829102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,768,0.00636053333679835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,768,0.015046399831771851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,768,0.005772800246874491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,512,0.003982933362325033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,512,0.015011200308799743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,16384,0.04058133363723755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,256,0.003306666761636734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,256,0.01478506624698639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,7168,0.02104746699333191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,128,0.003147733211517334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,6144,0.018926932414372762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,128,0.014712533354759217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,64,0.0029525332152843474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,128,0.004801066716512045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,6144,32,0.002868266652027766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,64,0.014716800053914389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,6144,32,0.014509866635004679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,65536,0.13861653010050456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,65536,0.2404970645904541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,4096,0.014045866330464682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,65536,0.12427626450856526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,51200,0.10755626360575359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,3584,0.01297813355922699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,51200,0.19051626523335774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,16384,0.06344960133234659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,16384,0.04558933178583781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,51200,0.09961919784545899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,12288,0.04882986545562744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,12288,0.037487999598185225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,10240,0.04134933153788249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,10240,0.033579734961191814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,1024,0.006307200094064077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,8192,0.03370560010274251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,8192,0.03009919921557109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,8192,0.020537600914637247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,512,0.005471999943256378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,7168,0.02821439901987712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,7168,0.028733867406845092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,6144,0.025041067600250246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,6144,0.02643839915593465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,6144,0.017017600933710735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,5120,0.021245867013931274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,5120,0.024229333798090617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,5120,0.014788267016410828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,4096,0.017828265825907387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,256,0.0049781332413355505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,6144,2560,0.010899200042088827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,4096,0.022358399629592896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,3584,0.01594986617565155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,3584,0.021861332654953002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,3072,0.014065066973368326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,3072,0.020773333311080933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,3072,0.011000532905260723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,2560,0.012133333086967468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,2560,0.02036693294843038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,2048,0.011638399958610535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,2048,0.01884373426437378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,2048,0.009001599748929341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,1536,0.008664533495903015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,1536,0.01662720044453939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,1536,0.007382399837176005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,1024,0.00680320014556249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,1024,0.015518933534622192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,1024,0.006446933249632518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,16384,0.03572479883829753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,768,0.004973866542180379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,12288,0.028753066062927247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,768,0.015210666259129844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,512,0.003688533355792364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,10240,0.02472106615702311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,512,0.005371733506520589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,256,0.003319466610749563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,256,0.01495146652062734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,256,0.0051242664456367494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,128,0.003099733342727025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,128,0.004851200183232625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,128,0.014563199877738953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,64,0.002833066632350286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,5120,32,0.0028757333755493166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,64,0.014503467082977294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,32,0.014691199858983359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,65536,0.18495893478393555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,7168,0.018899200359980266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,65536,0.11030720074971516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,51200,0.08956053256988525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,51200,0.0860970656077067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,16384,0.05026880105336508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,4096,0.012664533654848733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,16384,0.03969599803288777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,16384,0.031049599250157673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,12288,0.03955093224843343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,12288,0.03373013337453206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,5120,512,0.015064533551534018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,3584,0.012156800429026286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,10240,0.03344106674194336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,8192,0.027734400828679402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,10240,0.030779733260472612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,8192,0.027075199286142985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,7168,0.024988800287246704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,7168,0.025571199258168538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,2560,0.010256000359853109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,6144,0.02246613303820292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,6144,0.024205867449442545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,51200,0.1468789259592692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,5120,0.021526400248209634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,5120,0.02258560061454773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,5120,0.013291733463605246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,4096,0.015939199924468996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,4096,0.021192532777786256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,4096,0.011614933609962463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,3584,0.014429866274197897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,3584,0.02009493311246236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,5120,768,0.005673600236574808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,3072,0.011946666240692138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,3072,0.01990933418273926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,3072,0.010156800349553425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,2560,0.01042133371035258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,2560,0.018899200359980266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,65536,0.107204270362854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,12288,0.025085866451263428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,2048,0.009213866790135701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,10240,0.02172266642252604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,2048,0.008356266220410665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,2048,0.017794134219487508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,1536,0.00767573316891988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,1536,0.015637333194414772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,1536,0.0069930667678515124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,8192,0.017926400899887084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,1024,0.0060479998588562015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,768,0.003914666672547659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,1024,0.005682133138179779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,1024,0.015332266688346863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,512,0.0035434665779272715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,7168,0.016812799374262492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,768,0.01530346671740214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,512,0.014924800395965577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,256,0.0031946666538715364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,256,0.01461120049158732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,6144,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,128,0.003005866706371307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,128,0.014506666858990987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,64,0.002799999962250392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,64,0.014685866236686707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,4096,32,0.0028917332490285236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,4096,32,0.014729600151379904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,65536,0.169868803024292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,3584,0.011242666840553283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,65536,0.10229547023773193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,51200,0.13395412762959796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,65536,0.10334080060323078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,51200,0.08417173226674399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,51200,0.08220586776733399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,16384,0.04823999802271525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,16384,0.03645546833674113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,2560,0.00960213343302409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,12288,0.03470293283462524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,12288,0.03174933393796285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,12288,0.024052266279856363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,10240,0.03014613389968872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,10240,0.02096959948539734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,10240,0.02876586715380351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,8192,0.02478933334350586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,8192,0.0258026659488678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,8192,0.017628800868988038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,7168,0.022500266631444298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,7168,0.016122666994730632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,6144,0.02005973259607951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,6144,0.014628266294797262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,6144,0.023082667589187623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,5120,0.01744426687558492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,5120,0.021934932470321654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,5120,0.013152000308036805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,4096,0.014658133188883463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,4096,0.011530666550000509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,4096,0.020107734203338622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,3584,0.013143466909726462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,3584,0.019478400548299156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,3072,0.01088746686776479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,3072,0.019027199347813925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,3072,0.00993173321088155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,2560,0.00965119997660319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,2560,0.018396800756454466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,2560,0.009380267063776652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,2048,0.00844373305638631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,768,0.00557226687669754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,2048,0.01646719972292582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,1536,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,2048,0.007179733117421467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,1536,0.006793599824110668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,1536,0.015525333086649575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,512,0.005202133456865946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,1024,0.005207466582457224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,256,0.004958933095137278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,1024,0.006043733159701029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,768,0.004029866556326548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,1024,0.015490133563677469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,512,0.003538133452335993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,768,0.015225600202878317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,4096,128,0.004717866579691568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,512,0.014843733112017313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,256,0.0032511999209721885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,128,0.0029994666576385496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,256,0.005029333134492239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,128,0.014596266547838846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,7168,0.02417280077934265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,64,0.002807466685771942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3584,32,0.0030229332546393077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,64,0.014582399527231851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,32,0.014637866616249084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,65536,0.14453546206156415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,65536,0.08969066937764486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,65536,0.09525333245595297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,51200,0.11461866696675618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,51200,0.07229440212249756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,51200,0.07581973075866699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,16384,0.04005866845448812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,16384,0.035282135009765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,12288,0.03209919929504394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,12288,0.02985919912656148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,512,0.005218133330345154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,16384,0.030034132798512775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3584,256,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,10240,0.02722240090370178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,12288,0.021822933355967203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,10240,0.019757866859436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,10240,0.027319467067718504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,8192,0.016908800601959227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,8192,0.0249674657980601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,7168,0.020812799533208214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,7168,0.02446720004081726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,7168,0.015373866756757101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,6144,0.018454400698343913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,6144,0.022434133291244506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,5120,0.01608746647834778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,5120,0.020959999163945517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,3584,0.011057066917419433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,4096,0.013898666699727377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,4096,0.019870932896931967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,8192,0.023095466693242393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,768,0.005703466633955637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,3584,0.012633599837621055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,3584,0.010267733534177145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,3584,0.019061332941055296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,3072,0.011400533715883891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,3072,0.018478933970133463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,2560,0.009081600109736125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3584,128,0.004814933240413666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,2560,0.01720106601715088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,2048,0.007747200131416321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,2048,0.016536532839139303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,2048,0.00761599987745285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,1536,0.006740266581376393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,1536,0.01579839984575907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,1536,0.006614399949709575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,1024,0.004648533463478088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,1024,0.015292800466219583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,1024,0.005891199906667074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,768,0.003969066590070724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,768,0.015056000153223673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,768,0.005577600002288819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,16384,0.028013867139816285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,512,0.003606399893760681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,512,0.0052490666508674625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,512,0.014944000045458474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,256,0.003139200061559677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,256,0.014450132846832275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,128,0.002998399982849757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,128,0.014417066176732381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,128,0.004866133133570353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,64,0.002810666710138321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,64,0.014621866742769876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,3072,32,0.0028223998844623564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,3072,32,0.014498133460680643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,65536,0.12081066767374675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,6144,0.01399786671002706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,65536,0.07712746461232503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,5120,0.012563199798266093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,51200,0.06240533192952474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,16384,0.03339840173721313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,51200,0.07412160237630208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,16384,0.030857600768407184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,16384,0.027616000175476073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,12288,0.026521599292755126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,12288,0.022117332617441813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,12288,0.026822400093078614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,10240,0.023081600666046143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,10240,0.02482773264249166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,10240,0.01931519905726115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,8192,0.01920426686604818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,4096,0.011076266566912334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,8192,0.02333013415336609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,7168,0.017426133155822754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,7168,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,7168,0.022500266631444298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,6144,0.015550933281580605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,6144,0.021153066555658975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,5120,0.013610666990280152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,5120,0.020722132921218873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,5120,0.012366933623949687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,4096,0.011829333504041036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,4096,0.019221333662668864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,4096,0.010869333148002624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,3584,0.010856533050537109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,3072,0.00925546685854594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,3584,0.018611200650533042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,3072,0.010084266463915508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,3072,0.01784320076306661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,65536,0.09361066818237304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,2560,0.008303999900817871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,2560,0.01618133286635081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,2560,0.008376533786455791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,2048,0.0071487997968991595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,2048,0.016004266341527303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,8192,0.016515200336774193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,1536,0.005966933568318685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,1536,0.01560533344745636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,256,0.005076266825199127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,1024,0.004500266909599304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,1024,0.015282133221626281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,3072,2560,0.008736000458399455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,51200,0.09610026677449544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,768,0.003866666555404663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,768,0.015237333377202353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,768,0.005512533088525137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,512,0.003583999971548716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,512,0.014878933628400167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,6144,0.013677866260210673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,256,0.0031680000325044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,256,0.014679466684659323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,3584,0.01000853379567464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,128,0.0029493334392706556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,128,0.014519466956456503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,128,0.004833066463470459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,3072,0.009366400043169658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,64,0.00271573339899381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2560,32,0.0027594665686289472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,64,0.01456106702486674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2560,32,0.014469333489735923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,65536,0.09645439783732096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,65536,0.06338133414586386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,2048,0.007143466671307881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,51200,0.0759498675664266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,51200,0.0532970666885376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,16384,0.027523199717203777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,1536,0.006617600222428639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,16384,0.027543467283248902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,12288,0.021989333629608154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,16384,0.025382399559020996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,12288,0.024279467264811196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,10240,0.019221333662668864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,10240,0.023144533236821495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,1024,0.005981866518656413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,8192,0.01623466710249583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,8192,0.015413332978884378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,8192,0.02151573300361633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,7168,0.014603733023007711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,7168,0.020781866709391274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,6144,0.012995200355847678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,512,0.005415466427803039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,6144,0.020598399639129638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,5120,0.011760000387827556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,6144,0.012987732887268066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,5120,0.011657599608103435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,5120,0.01971093416213989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,4096,0.010305066903432209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,51200,0.06697173118591308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,3584,0.009693866968154908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,3584,0.017307732502619425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,3584,0.009296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,3072,0.00894719958305359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,3072,0.016474666198094685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2560,256,0.005016533533732096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,2560,0.007656533519426982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,2560,0.016012799739837647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,4096,0.01850559910138448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,2048,0.006297599772612255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,65536,0.08568533261617026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,2048,0.01593493322531382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,2048,0.007057066758473713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,1536,0.01583253343900045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,1536,0.006426666676998138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,1024,0.0041002665956815084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,1024,0.015294933319091797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,12288,0.020441599686940513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,768,0.003770666569471359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,768,0.01511679987112681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,768,0.005416533350944519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,512,0.00345920001467069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,10240,0.01793173352877299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,512,0.014852266510327658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,256,0.003151999910672506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,256,0.014604799946149192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,4096,0.010290132959683736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,7168,0.014148267110188803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,128,0.00290133332212766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,128,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,64,0.0026730666557947796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,1536,0.005029333134492239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,64,0.014613333344459533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,2048,32,0.002762666592995326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,2048,32,0.014602667093276978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,65536,0.07464426358540853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,65536,0.05380586783091227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,51200,0.05951146682103475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,51200,0.04531306823094686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,3072,0.008151466647783916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,16384,0.02234986623128255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,512,0.005276800195376078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,16384,0.024493867158889772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,12288,0.01803626616795858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,12288,0.022252800067265828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,2560,0.007947733501593272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,12288,0.02005973259607951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,10240,0.015615999698638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,10240,0.021342933177947998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,8192,0.013174399733543396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,8192,0.02023786703745524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,65536,0.0844661315282186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,7168,0.012277332941691081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,1024,0.0058005332946777345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,7168,0.019798400004704793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,256,0.004970666766166687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,6144,0.011237333218256634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,6144,0.019287467002868652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,2048,128,0.004808533191680908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,5120,0.01002346674601237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,5120,0.019058134158452353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,5120,0.01125759979089101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,4096,0.008974933624267578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,4096,0.017504000663757326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,4096,0.009332266449928284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,3584,0.008641067147254943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,3584,0.016620799899101257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,51200,0.06656320095062256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,3584,0.008955732981363932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,3072,0.008011733492215473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,3072,0.01676266590754191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,3072,0.008137600123882293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,2560,0.007187200089295705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,2560,0.0162090669075648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,2048,0.005635199944178263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,2048,0.01599253316720327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,16384,0.02481279969215393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,2048,0.006909866631031036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,1536,0.004735999802748362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,1536,0.015554133057594299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,1024,0.004117333392302195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,1024,0.015150933464368185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,1024,0.005786666770776113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,10240,0.017663999398549398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,768,0.003684266656637192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,768,0.005401599903901418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,768,0.015010133385658264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,512,0.003421866645415624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,512,0.014822399616241455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,256,0.0030389333764712016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,8192,0.015190399686495461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,256,0.014660267035166421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,128,0.002841600030660629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,128,0.014526933431625366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,7168,0.014043733477592468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,64,0.002665599932273229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,64,0.014618666966756186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1536,32,0.00266239990790685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1536,32,0.014403200149536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,65536,0.05032213528951009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,65536,0.04136213461558024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,51200,0.04052586555480957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,65536,0.08416533470153809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,51200,0.03558506568272908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,16384,0.016216533382733662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,6144,0.012723199526468911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,16384,0.021703465779622396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,12288,0.013473066687583923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,12288,0.02027413249015808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,2560,0.007650133470694225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,10240,0.012106666962305706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,10240,0.019633066654205323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,10240,0.017409066359202065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,8192,0.010436266660690308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,1536,0.006550399959087372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,8192,0.01923519968986511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,8192,0.015029333035151162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,7168,0.009827199578285217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,7168,0.018382932742436728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,6144,0.008897067109743754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,256,0.004950400193532308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,6144,0.017460266749064125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,5120,0.008494933446248371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,512,0.005096533397833506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,5120,0.018757333358128868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,4096,0.007820799946784973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,4096,0.017453867197036742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,4096,0.009297066926956176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1536,128,0.004747733473777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,3584,0.007119999825954437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,3584,0.01686613361040751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,3072,0.006534400085608165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,3072,0.016667733589808144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,3072,0.00809386670589447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,2560,0.006119466821352641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,2560,0.01636373301347097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,51200,0.06678400039672852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,5120,0.010558933019638062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,2048,0.005437866846720377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,16384,0.02488853335380554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,2048,0.01606186628341675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,1536,0.0047189335028330484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,1536,0.015607466300328573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,1024,0.004086400071779886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,12288,0.01993066668510437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,1024,0.005798399945100148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,1024,0.015179733435312906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,768,0.0036864000062147772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,768,0.015086932977040609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,512,0.0033514666060606635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,512,0.014838400483131408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,7168,0.013745066523551942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,256,0.0031199999153614043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,6144,0.011730133493741354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,256,0.014513066411018372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,256,0.004880000154177348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,128,0.0028405333558718365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,128,0.014318933089574179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,3584,0.008873599767684936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,64,0.0027402666707833606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,64,0.014499200383822122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,1024,32,0.0027232001225153605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,1024,32,0.014574933052062988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,65536,0.03965013424555461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,2560,0.0078015998005867004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,65536,0.03614826599756877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,51200,0.032545065879821776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,51200,0.03131626645723979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,2048,0.0069354668259620665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,16384,0.013421866297721862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,16384,0.020652800798416138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,768,0.0054058666030565895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,12288,0.011507200201352437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,12288,0.019428267081578573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,512,0.00510506679614385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,10240,0.010528000195821126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,10240,0.01835413376490275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,8192,0.009125333031018574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,8192,0.017148800690968833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,1536,0.006445866823196411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,7168,0.008669867118199667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,7168,0.01778879960378011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,1024,128,0.004830933113892873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,6144,0.008276266853014629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,6144,0.017588265736897788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,6144,0.011658666531244914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,5120,0.007241599758466085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,5120,0.01793280045191447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,5120,0.010487467050552368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,4096,0.00624533345301946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,65536,0.08414186636606852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,4096,0.017043199141820273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,3584,0.006563200056552887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,3584,0.01686720053354899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,3584,0.008830933769543966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,3072,0.006135466694831848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,51200,0.0662506659825643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,3072,0.01641279955705007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,2560,0.0060703997810681665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,3072,0.008012799918651581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,2560,0.00767573316891988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,2560,0.015936000148455302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,2048,0.005456000069777171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,2048,0.015599999825159708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,1536,0.004683733483155568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,1536,0.015401599804560342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,16384,0.02485760052998861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,1024,0.00410453329483668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,1024,0.015288533767064414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,1024,0.005798399945100148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,768,0.003651199986537298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,12288,0.01986879905064901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,768,0.015250133474667868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,512,0.0033845332761605583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,512,0.014755200346310934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,10240,0.017129600048065186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,256,0.003019733230272929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,256,0.014706133802731832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,8192,0.01464959979057312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,128,0.002811733384927114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,128,0.014429866274197897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,7168,0.013332266608874002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,64,0.002754133443037669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,64,0.014461867014567056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,768,32,0.0027690666417280836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,768,32,0.014512000481287637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,65536,0.028168533245722455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,65536,0.029183999697367353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,65536,0.08412160078684489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,51200,0.02332373261451721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,51200,0.025832533836364746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,51200,0.06649386485417684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,4096,0.009267200032869976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,16384,0.010568533341089885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,16384,0.019361066818237304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,16384,0.024268800020217897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,12288,0.009204266468683879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,12288,0.017384533087412515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,10240,0.008635733524958293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,10240,0.01757546663284302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,2048,0.006851199766000111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,8192,0.007365333537260692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,8192,0.017537067333857216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,1536,0.006438399851322174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,7168,0.006685866912206013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,7168,0.01767680048942566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,7168,0.013004799683888754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,6144,0.006247466802597046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,6144,0.01739306648572286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,768,0.005427200098832448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,5120,0.006346666812896728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,5120,0.018179200092951455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,512,0.0051701332132021586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,5120,0.010550399621327717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,4096,0.005994666616121928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,256,0.004902400076389313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,4096,0.01740586757659912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,3584,0.0064640000462532045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,3584,0.01691306630770365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,3584,0.00893440047899882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,3072,0.006073600053787232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,3072,0.016664533813794454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,3072,0.00800960014263789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,2560,0.0061589335401852924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,2560,0.01618026693662008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,2560,0.007703466713428498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,2048,0.005460266768932342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,2048,0.015897599856058757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,768,128,0.004686933259169261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,1536,0.004810666541258494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,2048,0.0068351998925209045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,1536,0.01548799971739451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,1024,0.004138666639725367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,1024,0.015106133619944253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,1024,0.005748266478379568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,768,0.0038304001092910765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,768,0.014934399724006652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,768,0.005430399874846141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,512,0.003436800092458725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,512,0.014860799908638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,512,0.005029333134492239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,256,0.003173333406448364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,256,0.014614400267601014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,12288,0.019492266575495402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,128,0.0029088000456492106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,128,0.014575999975204468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,128,0.00469760000705719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,10240,0.01692906618118286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,64,0.002679466704527537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,512,32,0.0026869334280490874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,64,0.014331733187039694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,65536,0.017630932728449504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,512,32,0.014629333217938741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,8192,0.014272000392278036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,65536,0.08398400147755941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,51200,0.013900799552599588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,51200,0.021826134125391642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,16384,0.007880533238252004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,51200,0.06598933140436808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,16384,0.01775040030479431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,12288,0.006487466891606649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,12288,0.01741973360379537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,6144,0.011667199929555257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,10240,0.0063381334145863845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,10240,0.017800533771514894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,4096,0.009130666653315227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,8192,0.0061941335598627726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,8192,0.01754666765530904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,7168,0.00602346658706665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,7168,0.017848533391952515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,65536,0.02332693338394165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,6144,0.005913599828879039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,6144,0.017373865842819212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,6144,0.011739733815193176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,5120,0.006270933151245117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,5120,0.018305067221323648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,5120,0.010473600029945374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,1536,0.00636053333679835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,4096,0.005904000004132589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,4096,0.009172266721725464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,4096,0.01718613306681315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,3584,0.006341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,3584,0.016897066434224447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,3072,0.006000000238418579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,3072,0.016748799880345663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,3072,0.007994666695594788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,2560,0.005976533393065134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,2560,0.01622293293476105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,512,256,0.004835199813048045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,2048,0.005339733262856802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,2048,0.015768532951672874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,2048,0.006775466601053874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,1536,0.004738133152325948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,1536,0.015464533368746439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,16384,0.0241376002629598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,1024,0.004058666775623957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,1024,0.015190399686495461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,1024,0.005723733206590017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,12288,0.01901866594950358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,768,0.003684266656637192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,768,0.015453867117563882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,512,0.003316266586383184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,512,0.014878933628400167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,512,0.005003733436266581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,10240,0.01664959987004598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,256,0.0029535998900731405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,256,0.014375467101732889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,128,0.0028213332096735638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,128,0.014266666769981385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,8192,0.014042666554450989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,64,0.0026154667139053345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,128,0.004806399842103322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,256,32,0.002595199892918269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,64,0.014562132954597472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,256,32,0.0144896000623703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,65536,0.011800533533096314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,7168,0.012877866625785828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,65536,0.02072319984436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,51200,0.010196266571680705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,65536,0.0833952029546102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,51200,0.01887893279393514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,16384,0.005975466469923655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,51200,0.0661130666732788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,16384,0.017655466000239053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,12288,0.005815466741720835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,12288,0.01725546717643738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,3584,0.00881706674893697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,10240,0.006038400034109751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,10240,0.017525333166122436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,10240,0.01651306649049123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,8192,0.005974400043487549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,8192,0.017400532960891724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,2560,0.007492266595363617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,7168,0.005962666869163513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,7168,0.01769919991493225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,1536,0.0064735998709996535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,6144,0.005801600217819214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,6144,0.017323732376098633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,6144,0.01165013313293457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,5120,0.006186666587988535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,5120,0.017566933234532674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,5120,0.010418132940928141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,4096,0.005745066702365876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,4096,0.017220266660054526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,768,0.005382399757703145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,3584,0.006180266539255777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,3584,0.016743467251459757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,3584,0.008706133564313252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,3072,0.00591786652803421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,256,256,0.004840533435344696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,3072,0.008035199840863545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,2560,0.005718400080998739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,2560,0.016042666633923848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,2560,0.007470933099587758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,2048,0.005145599941412607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,2048,0.01567893326282501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,1536,0.0045514668027559916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,1536,0.015228799978892007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,1536,0.006422399977842967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,1024,0.0038719999293486277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,16384,0.023827199141184488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,1024,0.015140266219774882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,12288,0.01893333395322164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,768,0.003587199995915095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,768,0.014644267161687216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,768,0.0053375999132792154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,512,0.0033088001112143196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,512,0.014740266402562461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,512,0.004980266590913137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,256,0.0029077333708604175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,256,0.014552533626556396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,128,0.0027306665976842242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,128,0.014405333002408347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,3072,0.016391467054684958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,64,0.0027189334233601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,64,0.014615466197331747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,128,32,0.002716800073782603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,8192,0.014130133390426635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,128,32,0.014477866888046264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,65536,0.008697600166002909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,51200,0.007419733206431071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,65536,0.01954560081164042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,7168,0.01288426617781321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,16384,0.00591786652803421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,51200,0.01890986760457357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,12288,0.005807999769846598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,16384,0.0176746666431427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,10240,0.005986133217811584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,12288,0.017541333039601644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,10240,0.017565866311391197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,8192,0.005853866537412008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,7168,0.005704533557097117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,8192,0.01746666630109151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,6144,0.0056639999151229855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,7168,0.0178656001885732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,5120,0.005971199770768484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,6144,0.017310933272043864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,5120,0.017579734325408936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,4096,0.005593599875768026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,3584,0.006066133578618368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,4096,0.017195733388264973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,3584,0.016778665781021117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,2560,0.005639466643333435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,3072,0.016432000199953715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,2048,0.0050453335046768185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,2560,0.016192000110944113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,1536,0.00445119986931483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,2048,0.015853866934776306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,1024,0.00378560001651446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,1536,0.015723733107248943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,4096,0.009241599837938945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,768,0.00348693331082662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,1024,0.015064533551534018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,512,0.003190399954716364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,768,0.014936533570289613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,256,0.002918400118748347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,512,0.014837333559989929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,128,0.0027744000156720476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,256,0.014552533626556396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,64,0.0026591998835404714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,128,0.014501333236694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,32,0.002611200014750163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,64,0.014468266566594442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,65536,0.008896000186602275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,64,32,0.014548266927401224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,51200,0.007821866869926452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,65536,0.01915839910507202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,16384,0.005891199906667074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,51200,0.01858560045560201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,12288,0.0059008002281188965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,16384,0.01798400084177653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,10240,0.006031999985376993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,12288,0.017729065815607705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,8192,0.005896533528963724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,10240,0.01748266617457072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,7168,0.0057087997595469155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,8192,0.017177599668502807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,6144,0.0055978665749231975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,7168,0.01767359972000122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,5120,0.005942399799823761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,6144,0.017489065726598106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,4096,0.005523199836413065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,5120,0.017934934298197428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,2048,0.006783999999364217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,3584,0.006016000111897787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,4096,0.017132800817489625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,3072,0.005766400198141734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,64,3072,0.005716266731421152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,3584,0.016614400347073875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,2560,0.005676800012588501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,3072,0.016457600394884746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,2048,0.005086933573087057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,1536,0.00453653335571289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,2560,0.016312533617019655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,2048,0.015646933515866598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,1024,0.003857066730658213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,1536,0.015313067038853965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,768,0.0034986667335033415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,512,0.003234133372704188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,1024,0.015732266505559287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,768,0.014938666423161825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,256,0.0028970666229724885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,512,0.014538666605949402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,128,0.00278613343834877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,64,0.002631466587384542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,256,0.01437333325544993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,128,0.014356266458829245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,11,32,32,0.002681600054105123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,64,0.014317867159843446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,11,32,32,0.014225066701571146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,256,0.004910933474699656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,128,0.004563199977080027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,16384,0.7473930358886719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,16384,0.3947231928507487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,11,128,1024,0.005723733206590017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,12288,0.2998762766520182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,12288,0.567796262105306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,51200,1.0928085327148438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,51200,1.1817418416341146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,10240,0.250492795308431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,10240,0.23322240511576334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,10240,0.47740694681803386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,8192,0.24007786115010582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,8192,0.3903200149536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,7168,0.32990614573160804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,7168,0.18063573837280272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,51200,2.3393590291341146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,7168,0.1592352072397868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,6144,0.13781226476033528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,6144,0.2849952061971029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,12288,0.2689098676045736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,5120,0.133572260538737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,5120,0.27913281122843425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,5120,0.11597653230031331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,4096,0.19960959752400714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,4096,0.10992746353149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,3584,0.09836693604787192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,3584,0.1976042588551839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,3072,0.14480214118957518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,3072,0.08660159905751547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,3072,0.08291733264923096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,2560,0.12127040227254231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,2560,0.07488853136698405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,16384,0.35614401499430337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,2048,0.09722987016042074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,2048,0.06286293268203735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,2048,0.054059731960296634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,1536,0.07483733495076497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,1536,0.051659735043843594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,1536,0.039901868502298994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,1024,0.05078399976094564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,1024,0.039622398217519124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,768,0.0398144006729126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,768,0.03389866749445598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,6144,0.15712000528971354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,768,0.023000532388687135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,512,0.03142613371213277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,512,0.027646932999293012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,512,0.017054933309555053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,256,0.02099840044975281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,256,0.012070399522781373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,128,0.00906773308912913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,128,0.01804800033569336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,128,0.009410132964452107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,64,0.006353066861629486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,64,0.016293332974116007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,32,0.004587733248869578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,65536,32,0.016382933656374613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,8192,0.18088639577229818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,65536,256,0.01544319987297058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,4096,0.10278933048248291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,3584,0.08362133502960205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,2560,0.06173973480860392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,65536,1024,0.028895999987920123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,65536,1.2039231618245443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,16384,0.608290163675944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,65536,1.1540949503580729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,16384,0.3180416107177734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,65536,2.3540308634440104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,16384,0.3452458699544271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,51200,0.9496863683064779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,12288,0.2237226645151774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,12288,0.2749706586201986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,12288,0.4631391843159993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,51200,0.9049599965413412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,51200,1.856984583536784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,10240,0.20072959264119467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,8192,0.16806613604227702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,10240,0.18667306900024414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,7168,0.26660799980163574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,8192,0.30540692011515297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,10240,0.4495039939880371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,7168,0.13808639844258624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,7168,0.16590506235758465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,6144,0.22272319793701173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,6144,0.12903146743774413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,5120,0.19197227160135905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,5120,0.09719786643981934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,4096,0.09204800128936767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,4096,0.07897706826527914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,4096,0.17551466623942058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,3584,0.08085439999898275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,3584,0.07038613160451254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,3584,0.13314133485158283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,3072,0.11466986338297527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,3072,0.07927040259043375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,2560,0.05208959976832071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,2560,0.09437119960784912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,2560,0.06250773270924886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,2048,0.08012479941050211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,1536,0.059195733070373534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,2048,0.05797866582870483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,1536,0.04375253518422444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,1536,0.033872000376383465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,1024,0.04068160057067871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,1024,0.033801599343617754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,1024,0.02488213380177816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,768,0.02863679925600688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,768,0.01976319948832194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,512,0.02180479963620504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,512,0.024194133281707764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,512,0.015178666512171427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,256,0.012319999933242797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,256,0.01935360034306844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,256,0.010812800129254658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,128,0.007593599955240886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,128,0.01579200029373169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,128,0.008030933141708375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,64,0.004183466732501984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,32,0.004203733305136363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,64,0.015843199690183003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,32,0.01609493295351664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,51200,5120,0.12199467023213703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,65536,0.3747690518697103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,51200,768,0.031557333469390866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,65536,0.7425173441569011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,8192,0.16441067059834796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,51200,0.29673493703206383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,16384,0.18349866867065429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,51200,0.31411946614583336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,51200,0.5873311996459961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,6144,0.1321450630823771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,16384,0.10668266614278157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,16384,0.09430932998657227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,12288,0.164466126759847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,12288,0.09693439801534018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,10240,0.11564479668935139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,8192,0.09378560384114584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,8192,0.05315093199412028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,3072,0.060842665036519375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,7168,0.08246719837188721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,7168,0.05578133265177408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,51200,2048,0.04835413297017415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,6144,0.07171306610107422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,6144,0.0564682682355245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,12288,0.07232000033060709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,10240,0.07238506476084391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,5120,0.059927467505137125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,5120,0.04416213432947795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,5120,0.03840106725692749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,4096,0.04892586469650269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,4096,0.03859093189239502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,3584,0.043295999368031815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,3584,0.03583360115687052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,3584,0.025549866755803424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,3072,0.03779093424479167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,3072,0.03290879925092061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,65536,0.3554464022318522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,2560,0.03701119820276896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,2560,0.03231786688168843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,2560,0.019806933403015137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,2048,0.026283733050028485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,2048,0.026580266157786053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,2048,0.016590933005015053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,1536,0.02068586746851603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,8192,0.06144533157348633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,1536,0.023516800006230673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,1024,0.014549332857131957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,1024,0.020462934176127115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,1024,0.010577066739400228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,10240,0.06842239697774252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,768,0.011603200435638427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,768,0.019144533077875774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,512,0.00862613320350647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,512,0.01732800006866455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,512,0.007684266567230225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,256,0.005670399963855743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,256,0.015066666404406228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,256,0.005437866846720377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,128,0.003588266670703888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,128,0.015000533064206442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,7168,0.044595201810201004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,64,0.0032672000428040824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,64,0.014988799889882406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,6144,0.03926506837209066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,16384,32,0.0032885332902272543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,16384,32,0.015056000153223673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,65536,0.2976703961690267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,65536,0.5540202458699544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,4096,0.02805759906768799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,51200,0.22017173767089843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,51200,0.23522772789001464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,51200,0.44022506078084306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,3072,0.022513065735499063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,16384,0.14400426546732586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,16384,0.09900053342183432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,12288,0.11083839734395344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,12288,0.06987199783325196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,12288,0.05957866509755453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,10240,0.09285226662953695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,10240,0.06028906504313151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,8192,0.07636586825052896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,8192,0.052001067002614344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,8192,0.040676267941792806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,7168,0.06654080152511596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,7168,0.04734613498051961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,7168,0.03600213527679443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,6144,0.057361066341400146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,6144,0.042626134554545086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,1536,0.013638400038083396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,6144,0.031856000423431396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,5120,0.04816533327102661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,5120,0.03828800121943156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,5120,0.027481599648793535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,768,0.00925546685854594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,4096,0.03393386602401734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,3584,0.03483946720759074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,3584,0.03132266600926717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,3584,0.021210666497548422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,3072,0.03037866751352946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,3072,0.029234133164087933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,3072,0.01875093380610148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,2560,0.025760000944137572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,2560,0.026477867364883424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,2048,0.02129279971122742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,2048,0.023991467555363973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,16384,128,0.005122133096059163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,1536,0.016521599888801575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,2048,0.013809067010879517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,1536,0.02156053384145101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,1536,0.011753599842389424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,1024,0.01188693344593048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,1024,0.019359999895095827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,1024,0.009294933080673218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,768,0.009706667065620423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,768,0.01827413241068522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,512,0.007406933108965556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,512,0.015133866667747497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,4096,0.03970880111058553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,256,0.0038794666528701783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,65536,0.28394667307535804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,256,0.015074132879575094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,128,0.0034677334129810332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,256,0.005228800078233084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,128,0.015024000406265258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,16384,0.08586666584014893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,64,0.003048533449570338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,64,0.014792533715566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,10240,0.04922879934310913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,12288,32,0.0030570665995279947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,12288,32,0.014838400483131408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,4096,0.023164800802866616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,65536,0.2592906634012858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,65536,0.47826347351074217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,2560,0.016722132762273155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,65536,0.2386613368988037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,51200,0.20253547032674155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,51200,0.37684691747029625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,16384,0.08352213700612386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,16384,0.12466239929199219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,768,0.008263466755549113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,12288,0.09516692956288655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,12288,0.05900373458862305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,12288,0.04899413188298543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,10240,0.07845226923624674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,512,0.005862399935722351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,10240,0.05156373182932535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,8192,0.04426666498184204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,8192,0.06723840236663818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,7168,0.05303039948145548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,7168,0.04160746733347575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,6144,0.04803626537322998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,6144,0.037887998421986896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,12288,128,0.004867200056711833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,5120,0.039026133219401044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,5120,0.033928533395131424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,4096,0.03189866741498311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,4096,0.030077866713205975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,16384,0.07455360094706218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,3584,0.02826559940973918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,10240,0.04202026526133219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,3584,0.028223999341328937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,3072,0.0249674657980601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,8192,0.03506666819254557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,3072,0.025863466660181682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,3072,0.016663466890652977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,2560,0.02127679983774821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,7168,0.03137173255284627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,2560,0.024370133876800537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,2048,0.017509333292643228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,2048,0.021916800737380983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,2048,0.012770133217175803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,1536,0.013801599542299906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,1536,0.020500266551971437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,1024,0.010124799609184266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,6144,0.02735146681467692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,1024,0.01857066750526428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,768,0.00853653351465861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,768,0.016749866803487144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,768,0.006982400019963582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,512,0.006626133124033611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,5120,0.02418559988339742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,512,0.015339733163515726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,256,0.003640533238649368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,256,0.014980266491572062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,256,0.005233066777388254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,4096,0.02057386636734009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,51200,0.18328107198079427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,128,0.014826666315396628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,64,0.0029898665845394133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,128,0.004907733201980591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,32,0.0029919999341169994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,64,0.01482133368651072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,10240,32,0.014884266257286071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,2560,0.014711466431617738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,1536,0.01088746686776479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,65536,0.19893226623535157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,65536,0.37556587855021156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,65536,0.1853482723236084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,51200,0.16021760304768878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,51200,0.14608960151672362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,16384,0.09248533248901367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,51200,0.2961941401163737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,12288,0.07083199818929037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,16384,0.05131306648254395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,16384,0.06094933350880941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,12288,0.055733335018157956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,10240,0.059178666273752845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,10240,0.0440778652826945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,10240,128,0.00335359995563825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,8192,0.04811306794484456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,8192,0.03853546778361003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,8192,0.028716800610224406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,7168,0.0426144003868103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,7168,0.02580053408940633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,7168,0.03527040084203084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,6144,0.03270293275515239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,5120,0.031516800324122116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,5120,0.029739733537038165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,5120,0.020041600863138834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,4096,0.025898667176564534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,3584,0.018732800086339315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,4096,0.026449066400527955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,1024,0.008770133058230083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,3584,0.0230240007241567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,3584,0.02490560015042623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,3584,0.015518933534622192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,3072,0.020514132579167683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,10240,512,0.005690666536490122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,3072,0.013799466689427695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,6144,0.037426133950551346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,2560,0.017450666427612303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,6144,0.022911999622980753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,2560,0.022292266289393105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,2048,0.014378666877746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,2048,0.02062293291091919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,2048,0.0110944002866745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,1536,0.011639466881752015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,1536,0.009723732868830364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,1536,0.019371734062830607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,1024,0.008643200000127155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,1024,0.01754986643791199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,1024,0.007928533355395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,768,0.007267199953397115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,768,0.015247999628384908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,768,0.0060127998391787205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,512,0.005790933469931285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,512,0.005548800031344095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,512,0.015277866522471109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,256,0.003626666714747747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,256,0.005109333495299021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,256,0.014974932869275412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,128,0.0031850665807724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,128,0.014647466937700906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,64,0.0030794667700926462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,64,0.014817066987355552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,8192,32,0.003047466774781545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,32,0.014888532956441245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,8192,3072,0.023222400744756063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,12288,0.04276373386383057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,10240,0.03422613143920898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,65536,0.3254197438557943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,65536,0.1841045379638672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,65536,0.1627434730529785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,51200,0.14929919242858886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,51200,0.2569354693094889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,16384,0.09175573190053304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,51200,0.13002453645070394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,16384,0.057643731435139976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,12288,0.06272533337275187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,4096,0.017485866943995156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,12288,0.050052265326182045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,12288,0.03624320030212402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,10240,0.05508373181025187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,10240,0.04079253276189168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,8192,0.045235200723012285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,7168,0.0377461314201355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,8192,0.03538879950841268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,8192,0.025963733593622845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,7168,0.03289600014686585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,6144,0.0309770663579305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,6144,0.021014400323232017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,5120,0.028360533714294433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,5120,0.027863466739654542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,4096,0.02332586646080017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,4096,0.025202133258183795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,2560,0.012733866771062216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,3584,0.020947200059890748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,3584,0.023988266785939537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,3584,0.014338133732477823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,3072,0.018475733200709023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,3072,0.022821333010991415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,3072,0.012780800461769104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,2560,0.01593280037244161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,2560,0.021647999684015908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,6144,0.033395198980967204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,2560,0.011750400066375732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,2048,0.013118933637936911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,2048,0.020166399081548055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,1536,0.010733866691589355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,8192,128,0.00487360010544459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,1536,0.009230933586756388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,1024,0.008062933385372163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,1024,0.01620693306128184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,16384,0.05264426469802856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,768,0.006919466455777486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,768,0.015502933661142984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,10240,0.030849067370096843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,768,0.006001066664854685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,512,0.004029866556326548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,7168,0.023425066471099855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,512,0.005390933156013489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,512,0.015278933445612588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,256,0.00346666673819224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,128,0.0031360000371932983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,256,0.014869333306948344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,128,0.004810666541258494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,128,0.014846932888031007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,64,0.0029311999678611755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,7168,32,0.002980266759792964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,64,0.014668800433476768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,5120,0.018257067600886027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,32,0.014452266693115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,4096,0.015560533603032431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,65536,0.1578154722849528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,65536,0.28207146326700844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,51200,0.1271018664042155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,51200,0.22218133608500162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,16384,0.08124053478240967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,16384,0.052026665210723876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,7168,1536,0.018757333358128868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,16384,0.040668801466623945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,12288,0.05785706837972006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,12288,0.04346880118052165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,10240,0.04977706670761108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,10240,0.03794879913330078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,2048,0.010237866640090942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,8192,0.03985706567764282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,8192,0.03374613523483276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,1024,0.006897066533565521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,7168,0.034417064984639485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,7168,0.03157973289489746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,7168,256,0.005228800078233084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,6144,0.030379732449849445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,6144,0.02924799919128418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,5120,0.025651200612386064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,5120,0.0264629324277242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,65536,0.14355626106262206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,5120,0.01651093363761902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,4096,0.021230934063593547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,4096,0.024254933993021647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,3584,0.019197867314020792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,3584,0.023566933472951253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,3584,0.013106133540471396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,3072,0.016985599199930826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,3072,0.021689599752426146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,51200,0.11476906935373943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,2560,0.0164192001024882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,2560,0.02071573336919149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,2560,0.011096533139546711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,2048,0.012521599729855856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,12288,0.03255040049552917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,2048,0.01945919990539551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,2048,0.009574400385220845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,1536,0.010065066814422607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,1536,0.018326399723688762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,1536,0.008781866232554118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,1024,0.007558399935563405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,1024,0.015065600474675497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,768,0.006426666676998138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,768,0.015125333269437154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,8192,0.02295680046081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,512,0.0038624001046021783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,512,0.015049599607785544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,7168,0.02108479936917623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,256,0.003271466741959254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,256,0.014829867084821067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,256,0.004997333387533823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,128,0.00311253344019254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,128,0.014518400033315023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,10240,0.02795413335164388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,64,0.002903466671705246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,64,0.014680533607800802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,6144,32,0.002887466549873352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,6144,0.01873706579208374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,6144,32,0.014762666821479798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,512,0.005392000079154968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,65536,0.13843946456909179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,65536,0.2404202620188395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,4096,0.013897599776585898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,51200,0.19025600751241048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,51200,0.09932586352030436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,16384,0.0454367995262146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,16384,0.0661621332168579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,16384,0.03570133447647095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,12288,0.04869439999262492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,3072,0.011778133114178975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,12288,0.037315201759338376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,12288,0.028523733218510944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,10240,0.04140053192774455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,10240,0.024651734034220378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,8192,0.03467626571655273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,8192,0.030199466149012248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,7168,0.028190932671229046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,7168,0.028358399868011475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,7168,0.01880426605542501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,1024,0.0064629331231117245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,6144,0.024911999702453613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,6144,0.026370133956273394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,5120,0.02128960092862447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,768,0.005998933315277099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,5120,0.024397865931193034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,5120,0.014774399995803832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,4096,0.017722666263580322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,4096,0.012748799721399941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,4096,0.022648533185323082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,3584,0.015869866808255514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,3584,0.021744000911712646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,3584,0.012090667088826498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,3072,0.014058666427930197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,3072,0.020846933126449585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,2560,0.012242133418718975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,51200,0.10762986342112224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,2560,0.019795199235280357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,6144,128,0.0048426667849222815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,2048,0.010359467069307963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,2048,0.018488534291585288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,2048,0.008889599641164144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,1536,0.008644266923268636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,10240,0.03370453516642253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,1536,0.016706132888793947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,1024,0.006843733290831248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,1024,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,65536,0.12420053482055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,768,0.0053610667586326596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,768,0.015128533045450846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,8192,0.020483199755350748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,512,0.0036202666660149893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,512,0.014915200074513755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,6144,0.016973867019017538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,256,0.0032298666735490165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,256,0.014893866578737893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,3072,0.010982400178909302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,128,0.0030559999247392017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,128,0.01458346645037333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,2560,0.010168533523877461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,128,0.00486826648314794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,64,0.014636799693107605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,32,0.0028245332340399425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,5120,32,0.01455573340257009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,1536,0.007755733529726664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,65536,0.10967679818471272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,1024,0.006234666705131531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,65536,0.18503146171569823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,65536,0.10735146999359131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,16384,0.05010666847229004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,51200,0.08926400343577066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,51200,0.14650452931722005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,16384,0.03935573498408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,16384,0.031052800019582112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,12288,0.03959146738052368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,12288,0.03378239870071411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,12288,0.024988800287246704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,10240,0.03349866469701131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,10240,0.021784534056981407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,10240,0.030898133913675945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,8192,0.027588266134262084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,7168,0.024970666567484538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,8192,0.027052799860636394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,7168,0.016687999169031777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,7168,0.025573333104451496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,6144,0.02226346731185913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,6144,0.02397226691246033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,6144,0.01490133305390676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,5120,0.019444266955057778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,5120,0.022473599513371786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,5120,0.013251200318336487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,4096,0.01592853367328644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,4096,0.02099626660346985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,5120,64,0.0028597332537174227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,3584,0.014437333742777506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,3584,0.020194133122762047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,3072,0.011925333738327026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,768,0.005719466507434845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,3072,0.010155733426411946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,3072,0.019504000743230186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,2560,0.010460799932479859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,2560,0.018985599279403687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,2048,0.008926933010419209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,2048,0.017528533935546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,2048,0.00844266712665558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,1536,0.007563733557860057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,1536,0.015923200050989787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,1536,0.007096533477306366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,1024,0.005986133217811584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,1024,0.015438933173815408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,1024,0.005901866654555003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,768,0.003967999915281932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,512,0.005425066749254862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,768,0.015110400319099427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,512,0.0035786665976047516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,768,0.0054506664474805195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,512,0.00525439977645874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,512,0.014932266871134438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,256,0.0031776001056035364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,256,0.014621866742769876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,256,0.004860800007979075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,5120,256,0.005089066425959269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,128,0.0028917332490285236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,64,0.002829866607983907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,128,0.01453439990679423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,4096,32,0.0028586665789286296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,64,0.01453013320763906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,4096,32,0.014640000462532044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,65536,0.16945385932922363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,65536,0.10203519662221272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,65536,0.1032639980316162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,51200,0.13503039677937825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,51200,0.08403840065002441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,51200,0.07842346827189127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,16384,0.048077865441640215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,16384,0.037010133266448975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,16384,0.029919999837875366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,12288,0.03563520113627116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,12288,0.03146986762682597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,51200,0.08571946620941162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,10240,0.033566931883494064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,10240,0.028665600220362346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,8192,0.018247467279434205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,10240,0.021121066808700562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,8192,0.024666666984558105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,8192,0.02582719922065735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,7168,0.02253546714782715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,7168,0.02443306644757589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,7168,0.01618666648864746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,6144,0.020170666774113975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,6144,0.023034665981928507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,3584,0.011186133821805317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,5120,0.017480534315109254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,5120,0.021894399325052896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,2560,0.009665066997210186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,5120,0.0130730668703715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,4096,0.014691199858983359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,4096,0.020406399170557657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,3584,0.013205333550771078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,3584,0.01961173415184021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,3584,0.011034666498502096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,3072,0.01093226671218872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,3072,0.01975040038426717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,3072,0.009990400075912476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,2560,0.009709866841634114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,2560,0.018617600202560425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,128,0.004729599754015604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,2048,0.008482133348782856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,2048,0.017343999942143758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,2048,0.008066133161385854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,1536,0.007190399865309398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,1536,0.015738667050997416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,12288,0.023426133394241332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,1024,0.004829866687456766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,1024,0.006121600170930227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,768,0.00402453343073527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,768,0.0150709331035614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,8192,0.017650133371353148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,4096,4096,0.011694932977358501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,512,0.003568000098069509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,512,0.005285333096981049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,512,0.014829867084821067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,256,0.003236266722281774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,256,0.014436266819636025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,128,0.002937600016593933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,128,0.01448319951693217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,128,0.004884266853332519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,64,0.0027776000400384264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,6144,0.014601600170135499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3584,32,0.002799999962250392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,64,0.014531200130780539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,32,0.014719999829928079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,65536,0.14427733421325684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,4096,0.011583999792734782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,65536,0.0890058676401774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,65536,0.09471999804178874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,51200,0.07211413383483886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,51200,0.11438186963399251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,16384,0.04007893403371175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,51200,0.07574293613433838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,16384,0.03480213483174642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,16384,0.028010666370391846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,12288,0.031956267356872556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3584,1024,0.015366400281588236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,12288,0.02247999906539917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,12288,0.029809067646662395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,10240,0.02723306616147359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,10240,0.02739306688308716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,8192,0.023101866245269775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,10240,0.019612799088160195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,8192,0.02521173357963562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,8192,0.015820800264676412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,7168,0.023931733767191567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,7168,0.01518186628818512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,6144,0.018309332927068076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,5120,0.01602773368358612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,2560,0.00949013332525889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,6144,0.022708266973495483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,4096,0.0139082670211792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,5120,0.012556800246238708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,5120,0.021142399311065672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,3584,0.012677333752314248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,4096,0.020385066668192543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,3584,0.01928960084915161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,3072,0.011461333433787028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,3072,0.01871253252029419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,3072,0.009409067034721375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,2560,0.009085866808891296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,2560,0.01755519906679789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,1536,0.006821333368619282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,2048,0.007934933404127757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,2048,0.016099199652671814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,2048,0.007355733215808869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,1536,0.006850133339564006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,1536,0.01564479966958364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,7168,0.020834134022394816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,1024,0.004391466577847799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,768,0.005619200070699056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,1024,0.015177599589029946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3584,256,0.005129600067933401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,768,0.003947733342647553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,768,0.015131733814875283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,768,0.005485866467158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,512,0.0035914666950702667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,512,0.01498133341471354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,6144,0.01386240025361379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,256,0.003236266722281774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,4096,0.010934399565060933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,256,0.014894933501879374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,3584,0.010682666301727295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,128,0.0029056000212828318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,128,0.014612266421318054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,64,0.002855466554562251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,64,0.014523733655611673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,3072,32,0.002863999952872594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,3072,32,0.014562132954597472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,2560,0.008553600311279297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,65536,0.12076586882273357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,65536,0.07668800354003906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,65536,0.0929738680521647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,51200,0.09595519701639811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,51200,0.0625109314918518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,51200,0.06865493456522623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,16384,0.033418667316436765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,1536,0.006647466619809468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,16384,0.027432533105214436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,16384,0.03072533408800761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,12288,0.026555732885996504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,12288,0.02722453276316325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,10240,0.024709333976109825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,10240,0.019339734315872194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,8192,0.019126399358113607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,8192,0.022784000635147093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,1024,0.005846400062243144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,8192,0.01634666621685028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,7168,0.01742186745007833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,7168,0.015192533532778421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,7168,0.022191999355951945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,6144,0.015565866231918335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,6144,0.021036799748738608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,5120,0.013563733299573263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,5120,0.020242132743199668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,5120,0.01225920021533966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,4096,0.011942399541536967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,512,0.0053375999132792154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,4096,0.01913386583328247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,4096,0.010941867033640544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,3584,0.010914132992426554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,3584,0.018658133347829182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,3072,0.01011199951171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,3072,0.01786880095799764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,10240,0.022954666614532472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,2560,0.00846506655216217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,2560,0.016302933295567833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,2048,0.007313066720962524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,2560,0.00860800047715505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,2048,0.007507200042406718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,2048,0.016085333625475564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,1536,0.005876266459623972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,1536,0.01585706671079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,1024,0.004378666480382284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,1024,0.015294933319091797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,1024,0.005854933460553487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,768,0.004043733328580856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,768,0.015077333648999533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,128,0.0047423998514811196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,768,0.005548800031344095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,512,0.01488746702671051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,512,0.0052159999807675685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,256,0.003206400076548258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,256,0.01453013320763906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,3584,0.010397866368293762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,128,0.0029535998900731405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,128,0.014574933052062988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,128,0.004855466882387797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,64,0.0027701333165168762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,64,0.014551466703414917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,32,0.0028021333118279776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2560,32,0.014550399780273438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,65536,0.09620479742685953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,12288,0.02050666610399882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,65536,0.06351573467254638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,3072,256,0.005087999999523163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,51200,0.07567359606424967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,51200,0.05328533252080282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,16384,0.027639466524124145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2560,512,0.003622400015592575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,16384,0.02765333255132039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,16384,0.025363200902938844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,12288,0.022222934166590373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,12288,0.024486400683720908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,12288,0.02040426731109619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,10240,0.019203199942906698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,10240,0.023156267404556275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,8192,0.01611733337243398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,8192,0.021683200200398763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,8192,0.015575466553370157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,3072,0.009165866176287334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,7168,0.01444906691710154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,7168,0.02076266606648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,7168,0.014315733313560485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,6144,0.01301653285821279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,6144,0.020070399840672812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,6144,0.01285653313000997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,5120,0.011733333269755047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,5120,0.019692800442377725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,5120,0.011702400445938111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,1536,0.00659093310435613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,4096,0.010220799843470256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,4096,0.018703999121983846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,4096,0.010337066650390626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,3584,0.009648000200589497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,6144,0.01376106639703115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,3584,0.017262933651606242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,3584,0.009501866499582927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,3072,0.009020800391832989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,2560,0.007643733421961467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,2560,0.016380799810091655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2560,256,0.00508480022350947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,2048,0.006135466694831848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,2048,0.01588159998257955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,65536,0.08444053332010905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,1536,0.00477866679430008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,3072,0.016457600394884746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,1536,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,1536,0.00650133341550827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,1024,0.004106666644414266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,1024,0.015335466464360556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,1024,0.005589333176612854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,768,0.0037834666669368743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,768,0.01493333379427592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,768,0.005364266534646353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,512,0.003368533402681351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,512,0.015013333161671957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,512,0.005121066669623057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,10240,0.017850667238235474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,256,0.003070933371782303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,128,0.0028789333999156954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,256,0.0145087997118632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,128,0.014509866635004679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,64,0.002701866626739502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,64,0.01460693379243215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,2048,32,0.0027424000203609467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,2048,32,0.014355199535687766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,65536,0.0744160016377767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,65536,0.05317546526590983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,65536,0.08354986508687337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,51200,0.05942080020904541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,51200,0.04451093276341756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,16384,0.02224853237469991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,51200,0.06758933067321778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,16384,0.024654932816823325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,51200,0.06820586522420248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,12288,0.018053332964579262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,12288,0.01994133392969767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,12288,0.022310400009155275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,10240,0.015370666980743408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,10240,0.02131519913673401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,10240,0.01755733291308085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,8192,0.013319466511408487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,3072,0.008690133690834045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,8192,0.019703465700149535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,7168,0.012290133039156596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,7168,0.02006933291753133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,7168,0.01395840048789978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,6144,0.010984533031781514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,2560,0.007810133198897044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,6144,0.01913386583328247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,5120,0.01023466686407725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,5120,0.01818880041440328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,4096,0.009134933352470398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,4096,0.01727786660194397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,2048,0.007017600039641063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,3584,0.008658132950464885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,3584,0.01660693287849426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,6144,0.01251413325468699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,3072,0.008164266745249431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,3072,0.016405333081881204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,256,0.00488319993019104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,2560,0.007098666826883952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,2048,128,0.004664533336957296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,2560,0.016401066382726034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,2048,0.00558186670144399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,2560,0.007652266820271809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,2048,0.015918933351834617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,1536,0.004722133278846741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,1536,0.01557866632938385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,16384,0.025116799275080363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,1024,0.0040832000474135075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,1024,0.015389866630236306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,1024,0.005717333157857259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,768,0.0038090666135152185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,768,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,8192,0.015101866920789084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,768,0.005492266515890757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,512,0.015009066462516785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,5120,0.01141866644223531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,4096,0.009525332848230999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,256,0.003110400090614955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,256,0.014642133315404256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,256,0.004860800007979075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,128,0.0028597332537174227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,128,0.014393599828084311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,3584,0.00902933379014333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,64,0.0027818667391935987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,64,0.014433067043622336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,32,0.002765866617361705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1536,32,0.014451199769973755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,65536,0.05037333170572916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1536,512,0.003332266708215078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,65536,0.040941866238911946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,51200,0.04041706720987956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,51200,0.03558506568272908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,51200,0.06680746873219809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,3072,0.008180266618728638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,16384,0.01607146660486857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,16384,0.021575466791788737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,16384,0.024782933791478477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,12288,0.013380266229311624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,12288,0.020308266083399452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,10240,0.012162133057912191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,2048,0.006903466582298279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,10240,0.02004800041516622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,8192,0.010434133807818095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,8192,0.019081600507100425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,1536,0.006419200201829274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,7168,0.00983786682287852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,7168,0.018054399887720743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,512,0.00510506679614385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,6144,0.008801066875457763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,6144,0.017661867539087932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,6144,0.012051199873288471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,5120,0.00858026643594106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,5120,0.01890346606572469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,5120,0.010643200079600016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,4096,0.008074666559696197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,4096,0.017486933867136636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,4096,0.009286399682362874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,3584,0.007255466779073079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1536,128,0.0047989333669344585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,3584,0.008983467022577922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,3072,0.006405333181222279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,3072,0.016721065839131674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,2560,0.0061141331990559895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,2560,0.016482133666674295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,2560,0.007715199887752533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,2048,0.005496533215045929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,2048,0.01597546637058258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,2048,0.006902400155862172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,1536,0.004743466774622599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,1536,0.015800533692042033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,65536,0.08411200046539306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,1024,0.004105599969625473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,1024,0.01506239970525106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,12288,0.01985599994659424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,768,0.00367253323396047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,768,0.014974932869275412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,768,0.005411200225353241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,10240,0.01739413340886434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,512,0.003420799970626831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,512,0.014682666460673014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,3584,0.017407999436060587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,8192,0.014945066968599954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,256,0.00296426663796107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,128,0.002762666592995326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,3072,0.008055466910203297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,256,0.0048213332891464235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,256,0.01452906628449758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,64,0.0027424000203609467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,128,0.004808533191680908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,128,0.014393599828084311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,1024,32,0.002749866743882497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,7168,0.013542399803797404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,64,0.01448319951693217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,51200,0.03218666712443034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,65536,0.03952213525772095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,65536,0.035709865887959796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,65536,0.0840554634730021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,16384,0.013547733426094055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,16384,0.020595200856526694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,16384,0.02481493353843689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,12288,0.011569066842397054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,12288,0.01986560026804606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,12288,0.019785600900650024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,10240,0.010525866349538168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,10240,0.01866133411725362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,10240,0.017064533631006875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,8192,0.009305600325266521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,8192,0.0173632005850474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,8192,0.014451199769973755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,7168,0.008679466446240743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,7168,0.017411200205485027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,6144,0.008243200182914735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,6144,0.017271467049916587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,1024,32,0.014403200149536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,5120,0.007417599856853485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,51200,0.03139946659406026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,5120,0.017890133460362754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,5120,0.01051093339920044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,4096,0.006369066735108693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,4096,0.01721386710802714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,1536,0.006424533327420552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,3584,0.006685866912206013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,3584,0.01700906753540039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,3584,0.009080533186594646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,1024,0.00576853354771932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,3072,0.016523733735084534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,1024,512,0.005155199766159057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,2560,0.00600853314002355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,2560,0.01616320013999939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,2560,0.00759680022795995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,2048,0.005379199981689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,2048,0.015947733322779337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,51200,0.0667253335316976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,1536,0.004638933142026265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,1536,0.015340800086657206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,1536,0.006428800026575724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,1024,0.003957333415746689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,1024,0.015109333395957946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,1024,0.005696000158786773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,768,0.0036906667053699495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,7168,0.013145599762598673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,3072,0.006101333101590474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,768,0.014906666676203408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,512,0.0034229333202044168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,6144,0.011781332890192668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,512,0.005197866757710775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,512,0.014739200472831726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,256,0.003066666672627131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,256,0.004855466882387797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,256,0.014582399527231851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,128,0.014451199769973755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,64,0.00262719988822937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,64,0.014376533031463624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,32,0.0026496000587940215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,768,32,0.014446933070818582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,65536,0.028201599915822346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,65536,0.028970666726430255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,51200,0.02310933272043864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,51200,0.026106667518615723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,4096,0.009326933821042379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,51200,0.0664789319038391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,16384,0.010626133282979329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,16384,0.018988800048828126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,16384,0.02448319991429647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,12288,0.009273599584897358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,12288,0.01733013391494751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,768,0.0053493330876032506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,10240,0.008545066912968953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,10240,0.017914666732152303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,768,128,0.0028618666032950084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,8192,0.00764160007238388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,8192,0.01432213286558787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,8192,0.017691733439763387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,7168,0.006640000144640605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,7168,0.01772693395614624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,6144,0.0062282666563987735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,3072,0.008051200211048127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,6144,0.017511467138926186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,5120,0.006377600133419037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,6144,0.01164479951063792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,5120,0.01816533406575521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,4096,0.005842133363087972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,4096,0.017452800273895265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,4096,0.009125333031018574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,3584,0.006427733103434245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,3584,0.01709866722424825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,2048,0.00689386675755183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,3072,0.006097066899140676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,3072,0.016908800601959227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,3072,0.007982933521270752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,2560,0.006141866743564606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,2560,0.016266666849454246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,10240,0.016921599706014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,2048,0.00551146666208903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,2048,0.015708800156911215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,2048,0.0068351998925209045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,1536,0.004747733473777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,1536,0.015574399630228677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,768,128,0.004744533201058706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,1536,0.006361599763234456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,1024,0.0041461333632469176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,1024,0.015296000242233276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,1024,0.005709866682688395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,768,0.003764266769091288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,768,0.015229866902033488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,768,0.0053951998551686605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,512,0.003492266684770584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,512,0.014837333559989929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,256,0.003205333401759466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,256,0.014729600151379904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,65536,0.08384746710459391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,128,0.002903466671705246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,128,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,128,0.004804266492525736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,64,0.0027146667242050173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,64,0.014516266187032065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,12288,0.01950613260269165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,512,32,0.002733866622050603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,512,32,0.014503467082977294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,65536,0.017505067586898803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,65536,0.023430399099985757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,7168,0.013044266899426778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,51200,0.01381013294061025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,51200,0.021901865800221763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,5120,0.01051520009835561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,16384,0.007815466821193695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,51200,0.06646506786346436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,16384,0.017893334229787193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,16384,0.024024534225463866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,12288,0.006493866443634033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,12288,0.017762132485707603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,12288,0.018913066387176512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,10240,0.006415999929110209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,10240,0.01768640081087748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,10240,0.016598400473594666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,8192,0.006196266909440359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,3584,0.008885332942008972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,8192,0.017517866690953572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,7168,0.006011733412742614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,7168,0.018209065993626913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,6144,0.005913599828879039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,6144,0.01763733426729838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,2560,0.007652266820271809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,5120,0.00636053333679835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,5120,0.017939200003941856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,5120,0.010461866855621338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,4096,0.005921066800753275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,4096,0.017352533340454102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,4096,0.009194667140642803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,3584,0.00626986672480901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,3584,0.016804265975952148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,512,0.005083733300367991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,3072,0.006074666480223338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,512,256,0.00491946687301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,3072,0.016703999042510985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,2560,0.006098133325576782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,2560,0.016302933295567833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,2560,0.0076799998680750535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,2048,0.005373866856098175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,2048,0.015964800119400026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,2048,0.006804266571998596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,1536,0.004689066608746847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,1536,0.015732266505559287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,1536,0.006404266754786174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,1024,0.004012800008058548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,1024,0.015269333124160766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,768,0.0036703998843828833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,768,0.015156267086664834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,65536,0.08347307046254476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,512,0.0033621333539485933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,512,0.01483519971370697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,8192,0.014097066720326743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,256,0.0030122667551040648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,7168,0.012828800082206725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,256,0.014591999848683677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,128,0.002833066632350286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,128,0.014497066537539164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,6144,0.011739733815193176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,64,0.002721066772937775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,64,0.014532267053922018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,256,32,0.002657066782315572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,256,32,0.01451520025730133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,65536,0.011698133746782939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,65536,0.020529067516326903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,65536,0.0832266648610433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,51200,0.010146133104960124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,3584,0.008875733613967896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,51200,0.018713599443435668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,16384,0.005986133217811584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,51200,0.06573226849238077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,16384,0.01760106682777405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,12288,0.005866666634877523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,12288,0.017372800906499227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,12288,0.01885439952214559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,10240,0.006056533257166544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,3072,0.008038400113582611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,10240,0.017921066284179686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,10240,0.016532267133394875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,8192,0.005937066674232483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,8192,0.014121599992116294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,8192,0.017754666010538735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,7168,0.005847466488679251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,7168,0.017627733945846557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,6144,0.005790933469931285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,6144,0.017339734236399333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,6144,0.011660800377527872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,5120,0.006162133316198985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,5120,0.01795626680056254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,1024,0.005745066702365876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,4096,0.0056981335083643595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,768,0.005352533360322317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,4096,0.017030400037765504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,4096,0.009324799974759419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,3584,0.006179200112819671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,3584,0.016832000017166136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,3072,0.005927466849486033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,512,0.005028266708056132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,3072,0.016555733482042947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,256,0.004790399968624115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,2560,0.005773866673310598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,2560,0.015945600469907124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,256,128,0.004717866579691568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,2048,0.005120000243186951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,2048,0.01616853376229604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,2048,0.006730666756629944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,1536,0.00450133333603541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,1536,0.015664000312487283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,1536,0.006409599880377452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,1024,0.003912533322970072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,1024,0.015210666259129844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,1024,0.005602133274078369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,768,0.0036309334139029183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,768,0.014863999684651694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,768,0.005345066885153452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,16384,0.02373653252919515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,512,0.014905599753061929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,7168,0.01291306714216868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,256,0.003013333429892858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,256,0.014472533265749613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,256,0.004808533191680908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,128,0.0027434666951497394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,128,0.014417066176732381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,5120,0.010450133681297302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,64,0.0026496000587940215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,64,0.014605866869290671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,32,0.0026335999369621276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,128,32,0.014595199624697366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,65536,0.008614400029182434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,128,512,0.003337600082159042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,3584,0.008866133292516072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,65536,0.019028266270955406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,51200,0.00746559997399648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,16384,0.0059008002281188965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,12288,0.005762133498986562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,16384,0.01747093399365743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,10240,0.005924266576766968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,12288,0.017432532707850137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,8192,0.005843199789524078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,10240,0.01755626598993937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,7168,0.005715199808279673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,8192,0.017364267508188883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,6144,0.005622399846712748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,3072,0.008021333316961924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,7168,0.01766080061594645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,5120,0.005939200023810069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,4096,0.005629866818586985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,6144,0.017146666844685875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,5120,0.017833600441614784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,3584,0.006041599810123444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,3072,0.005748266478379568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,3584,0.016545066237449647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,3072,0.016459733247756958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,2048,0.00506986677646637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,2560,0.015998933712641397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,1536,0.004522666831811269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,2048,0.01592639982700348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,1024,0.003835733234882355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,1536,0.015225600202878317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,768,0.0035434665779272715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,1024,0.015201066931088766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,512,0.0032170665760835014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,768,0.014717866977055868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,512,0.014879999558130899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,256,0.0028768000503381092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,2560,0.007582933207352956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,128,0.00276799996693929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,256,0.014571733276049294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,64,0.0026581334571043652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,128,0.014386133352915446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,32,0.0026975999275843303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,64,0.014628266294797262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,32,0.014311466614405313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,51200,0.008030933141708375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,65536,0.01949333349863688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,16384,0.00589333325624466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,51200,0.018900267283121743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,12288,0.005740800003210703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,16384,0.017502933740615845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,10240,0.00609493354956309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,12288,0.017154133319854735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,8192,0.005876266459623972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,10240,0.017722666263580322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,51200,0.018620800971984864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,7168,0.0057536001006762184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,8192,0.017068799336751303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,6144,0.005594666798909505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,5120,0.005970133344332377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,7168,0.017550933361053466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,4096,0.0055285334587097164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,5120,0.017593600352605186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,3584,0.006056533257166544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,4096,0.017219199736913045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,3072,0.005628799895445505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,3584,0.016682666540145875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,512,0.005073066552480062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,2560,0.005649066468079885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,64,4096,0.017053866386413576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,3072,0.016389333208402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,2560,0.01593706707159678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,64,2560,0.005657599866390228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,2048,0.0050794666012128195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,1536,0.004507733384768168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,1024,0.0038218667109807336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,2048,0.016076800227165223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,1536,0.015621333320935567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,768,0.0034805332620938623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,1024,0.015015467007954916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,512,0.003219199925661087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,768,0.01477013329664866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,256,0.0029365333418051405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,512,0.014779733618100485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,128,0.00275093341867129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,256,0.014432000120480857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,64,0.0026069333155949908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,128,0.014477866888046264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,32,0.002621866762638092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,64,0.014345600207646688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,32,0.014485333363215128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,10,32,65536,0.00883840024471283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,10,128,128,0.004569600025812784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,10,32,6144,0.017068799336751303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,16384,0.7465440114339192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,16384,0.39205331802368165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,16384,0.35656426747639974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,12288,0.5660405476888021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,12288,0.2978997230529785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,10240,0.48235626220703126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,51200,1.1805620829264323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,51200,1.0967989603678385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,8192,0.377401606241862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,8192,0.21125334103902182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,8192,0.24007573127746581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,7168,0.18239893913269042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,7168,0.3300863901774088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,51200,2.3398282368977865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,6144,0.2849887847900391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,6144,0.15693440437316894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,10240,0.2501333395640055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,5120,0.13826133410135905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,5120,0.27956266403198243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,4096,0.10936106840769451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,4096,0.19444692929585775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,4096,0.10845866998036702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,3584,0.16799039840698243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,3584,0.09850773016611734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,3584,0.09588267008463541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,3072,0.08674133618672689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,3072,0.14481600125630695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,3072,0.08294719854990641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,2560,0.0751477320988973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,2560,0.12108799616495769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,2560,0.07024746735890706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,2048,0.09692373275756835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,2048,0.0626421332359314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,1536,0.07465919653574625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,1536,0.05164586702982584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,12288,0.2684373219807943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,1024,0.05075519879659017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,1536,0.04450773398081462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,1024,0.028680533170700073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,768,0.039774934450785324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,1024,0.043535999457041424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,768,0.03362346490224202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,768,0.02280319929122925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,512,0.027428267399470012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,512,0.027142399549484254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,256,0.015477333466211954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,256,0.012176000078519185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,128,0.009118933478991191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,128,0.018245333433151247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,10240,0.22495253880818686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,128,0.009426132837931315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,64,0.006717866659164429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,64,0.016436266899108886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,65536,32,0.004669866462548574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,32,0.016375466187795003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,7168,0.15913813908894855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,6144,0.1418069362640381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,65536,256,0.02095253268877665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,5120,0.11581439971923828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,65536,1.2108927408854167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,2048,0.050842666625976564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,65536,512,0.017108267545700072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,51200,0.9628586451212564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,16384,0.3140351931254069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,65536,1.1543423970540365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,51200,0.9365301132202148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,65536,2.3651713053385417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,16384,0.6215050379435222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,12288,0.2395871957143148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,16384,0.2954655965169271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,51200,1.859491221110026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,12288,0.4909397443135579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,10240,0.40111147562662763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,10240,0.20458240509033204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,10240,0.22999146779378257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,8192,0.1661344051361084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,8192,0.3004821459452311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,7168,0.15798080762227376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,7168,0.2607903957366943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,6144,0.22193919817606605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,6144,0.12731413046518963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,5120,0.19411946932474772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,5120,0.0970240036646525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,5120,0.11114880243937175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,4096,0.08922773202260335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,4096,0.0788586695988973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,4096,0.1563722610473633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,3584,0.08065173625946045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,3584,0.13048853079477946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,3072,0.11495893001556397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,3072,0.07084266344706217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,3072,0.06083733240763346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,2560,0.09805440107981364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,2560,0.06145386695861817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,2048,0.07908693154652914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,2048,0.05249919891357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,2048,0.0427562673886617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,1536,0.059189331531524655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,1536,0.04399786790211995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,12288,0.259278933207194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,1024,0.046359467506408694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,1024,0.03676480054855347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,8192,0.17471680641174317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,1024,0.024774400393168132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,7168,0.14729065895080568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,768,0.01983893314997355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,768,0.035496532917022705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,512,0.024565333127975465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,6144,0.11521813074747722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,512,0.023982934157053628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,256,0.012247467041015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,128,0.007606400052706401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,256,0.02025493383407593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,128,0.016141866644223533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,64,0.004193066557248434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,64,0.01618880033493042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,51200,32,0.004161066561937332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,32,0.016125866770744325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,3584,0.07009279727935791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,2560,0.05214720169703165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,51200,768,0.028915200630823774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,65536,0.3751946767171224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,65536,0.741486930847168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,512,0.0160863995552063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,1536,0.03389546473821004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,65536,0.3550741195678711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,16384,0.1833610693613688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,16384,0.10699199835459391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,51200,0.29659414291381836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,16384,0.09472746849060058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,12288,0.13868692715962727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,51200,0.5884639739990234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,10240,0.11566826502482097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,10240,0.07240107059478759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,8192,0.09414827028910319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,8192,0.06150399843851725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,8192,0.04979519844055176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,7168,0.08266133467356364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,7168,0.05589439868927002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,7168,0.044377601146698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,6144,0.07157226403554282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,6144,0.050006401538848874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,6144,0.03910186688105265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,5120,0.06006933450698852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,5120,0.033690667152404784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,5120,0.04423040151596069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,4096,0.04889599879582723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,4096,0.03820480108261108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,4096,0.027879466613133747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,3584,0.04315306742986043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,256,0.010912000139554342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,3584,0.03583893378575643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,3584,0.025483733415603636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,51200,128,0.007973333199818928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,3072,0.03785813252131144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,3072,0.03283626635869344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,2560,0.032052266597747806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,2560,0.019870932896931967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,2560,0.029681066672007244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,2048,0.016578132907549538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,2048,0.0265120009581248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,1536,0.020565332969029744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,1536,0.023476266860961915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,1536,0.013648000359535218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,1024,0.02066346605618795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,1024,0.010603732864061991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,768,0.011665067076683045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,768,0.01917546590169271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,768,0.009282132983207703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,512,0.008643200000127155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,512,0.017590399583180746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,512,0.007602133353551229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,256,0.005694933235645294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,256,0.015411200126012168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,3072,0.02239039937655131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,2048,0.026162133614222212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,128,0.0035135999321937563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,12288,0.08424533208211263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,128,0.014891733725865683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,128,0.005013333261013031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,64,0.0032501332461833954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,64,0.014885333180427552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,32,0.00322026660044988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,51200,0.27878398895263673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,16384,32,0.015005866686503092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,12288,0.07237226963043213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,10240,0.07085013389587402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,65536,0.295635191599528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,51200,0.43537174860636396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,65536,0.2837429364522298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,65536,0.553881581624349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,51200,0.237280003229777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,16384,0.14458559354146322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,16384,0.09851306279500326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,16384,1024,0.014573867122332254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,16384,0.08706879615783691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,12288,0.11157120068868001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,12288,0.0701141357421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,10240,0.09275093078613281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,10240,0.06090240081151327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,10240,0.04816533327102661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,8192,0.05165760119756063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,8192,0.07626240253448487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,7168,0.06647253433863322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,7168,0.04709226687749227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,7168,0.035867734750111895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,6144,0.057340800762176514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,6144,0.04269973436991374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,6144,0.03185919920603435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,5120,0.04819200038909912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,5120,0.03826666673024495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,5120,0.02728319962819417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,4096,0.03958293199539185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,3584,0.0348853349685669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,3584,0.031204267342885332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,16384,256,0.005409066875775656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,3072,0.030294400453567506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,3072,0.028959999481836956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,3072,0.0186463991800944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,2560,0.025884799162546795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,2560,0.02630400061607361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,2560,0.016637866695721946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,2048,0.02134933272997538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,2048,0.023806933561960855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,2048,0.013808000087738036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,1536,0.016734933853149413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,1536,0.02183253367741903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,1024,0.011911466717720032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,1024,0.019397334257761637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,4096,0.033598931630452473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,768,0.009687466422716777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,768,0.01825173298517863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,768,0.00819413314263026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,512,0.007397333284219106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,512,0.015225600202878317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,51200,0.220306126276652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,256,0.0038880000511805216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,256,0.01464959979057312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,256,0.005303466816743215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,128,0.0034048000971476236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,12288,0.05782399972279867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,128,0.014964266618092855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,128,0.004925866425037384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,64,0.0030613332986831666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,12288,32,0.003089066594839096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,64,0.014856533209482829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,8192,0.04005333185195923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,12288,32,0.014967466394106546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,4096,0.022954666614532472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,65536,0.25834453900655113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,65536,0.23919572830200195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,3584,0.020935465892155967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,65536,0.47814292907714845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,16384,0.12450559933980306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,16384,0.08533013661702474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,51200,0.20230719248453774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,51200,0.3760714530944824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,16384,0.06273599863052368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,12288,0.09498559633890788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,12288,0.0658453345298767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,10240,0.07814506689707437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,10240,0.05148053169250488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,10240,0.04211626847585042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,8192,0.06717867056528727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,8192,0.0445525328318278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,8192,0.034701867898305254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,7168,0.05492586692174276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,7168,0.031267199913660684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,6144,0.048097066084543866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,6144,0.037164799372355145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,6144,0.027397332588831584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,5120,0.0389354666074117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,5120,0.034040534496307374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,1536,0.011901866396268208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,4096,0.03282986680666606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,4096,0.02946026722590129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,1024,0.009198932846387228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,3584,0.028321067492167156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,3584,0.027903999884923297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,12288,512,0.005793066819508871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,3072,0.024757333596547446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,51200,0.18336853981018067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,3072,0.025913600126902265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,2560,0.021050665775934854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,3072,0.018389334281285606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,2560,0.024099200963974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,2560,0.014637866616249084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,2048,0.017297067244847617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,2048,0.02217386762301127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,1536,0.013832533359527588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,1536,0.020258132616678873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,7168,0.0405898650487264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,1536,0.010757333040237427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,1024,0.01013973355293274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,1024,0.018472532431284584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,1024,0.00872320036093394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,768,0.008423466483751934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,768,0.016908800601959227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,768,0.006570666531721751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,512,0.006614399949709575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,512,0.00556160012880961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,512,0.01541973352432251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,256,0.003568000098069509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,256,0.005230933427810669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,128,0.0032853332658608755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,128,0.014677332838376364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,12288,0.05670719941457113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,64,0.003005866706371307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,64,0.014863999684651694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,10240,32,0.0030602666238943735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,32,0.014757333199183145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,5120,0.024115200837453207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,4096,0.020512000719706217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,65536,0.37563947041829426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,3584,0.01872746745745341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,65536,0.1987402598063151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,65536,0.18576213518778484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,16384,0.09236586888631185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,51200,0.16057066917419432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,51200,0.2959520022074381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,12288,0.07076799869537354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,12288,0.055956268310546876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,2048,0.012596266468365988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,10240,0.044151465098063156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,10240,256,0.015033599734306336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,10240,0.03440426588058472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,8192,0.0482367992401123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,8192,0.038182401657104494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,8192,0.02872640093167623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,7168,0.03516586621602376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,10240,128,0.004881066580613455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,6144,0.03725866476694743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,6144,0.03261973261833191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,16384,0.06075839996337891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,5120,0.03157653411229451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,5120,0.029736532767613726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,5120,0.020104533433914183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,4096,0.025685334205627443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,4096,0.02619733413060506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,4096,0.017399466037750243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,3584,0.022986666361490885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,3584,0.025066665808359784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,7168,0.04250986576080322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,3072,0.020438400904337566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,3072,0.023625600337982177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,3072,0.013844266533851624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,2560,0.017453867197036742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,2560,0.022258132696151733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,51200,0.14552000363667805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,2048,0.016521599888801575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,16384,0.051268267631530764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,2048,0.020807466904322305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,1536,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,2048,0.011199999849001567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,12288,0.03962773482004801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,1536,0.0192138671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,1024,0.008725333213806152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,1024,0.017679999272028603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,1024,0.007853866616884867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,768,0.007288533449172974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,768,0.0060032000144322716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,768,0.015560533603032431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,512,0.0056320001681645715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,512,0.015212800105412802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,512,0.005636266867319743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,256,0.0035264000296592714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,256,0.014973866939544677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,128,0.003260799994071325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,128,0.014724266529083253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,7168,0.02570880055427551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,64,0.002958933264017105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,64,0.014765866597493491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,32,0.0029738667110602063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,8192,32,0.014849066734313965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,6144,0.02313279906908671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,8192,10240,0.0589738647143046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,65536,0.32493438720703127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,1536,0.009815466403961182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,65536,0.1830933411916097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,51200,0.14794880549112957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,51200,0.2565738677978516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,256,0.005298133194446564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,3584,0.015609600146611533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,16384,0.04585493405659993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,16384,0.05742293198903402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,16384,0.0972202698389689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,12288,0.0626250664393107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,12288,0.04676053524017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,10240,0.05505280097325643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,10240,0.04137066602706909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,10240,0.030971733729044597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,8192,0.04636586507161458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,2560,0.012636799613634744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,8192,0.03596693277359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,7168,0.03795733451843262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,7168,0.033111466964085894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,7168,0.023244800170262654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,6144,0.03327573339144389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,6144,0.03090133269627889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,6144,0.02101226647694906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,5120,0.028146133820215864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,5120,0.02806719938913981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,4096,0.023222400744756063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,4096,0.025272534290949507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,4096,0.015485866864522298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,3584,0.020824533700942994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,3584,0.024264534314473472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,3584,0.014307199915250143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,3072,0.01845653255780538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,3072,0.022742400566736855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,3072,0.01281599998474121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,2560,0.015875200430552162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,2560,0.02135146657625834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,2560,0.011784533659617107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,2048,0.01316159963607788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,2048,0.020206934213638304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,2048,0.010305066903432209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,1536,0.010805333654085796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,8192,128,0.004878933231035868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,1536,0.01872746745745341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,1024,0.00811413327852885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,1024,0.015496533115704855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,1024,0.006584533552328746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,768,0.0068906664848327635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,768,0.01520746648311615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,768,0.005965866645177205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,512,0.004084266722202301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,512,0.01532586713631948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,256,0.0034058667719364167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,65536,0.17014293670654296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,256,0.01498240033785502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,256,0.00521066685517629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,128,0.0031221332649389905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,128,0.004880000154177348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,128,0.014582399527231851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,64,0.0030218665798505146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,51200,0.12979733149210612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,7168,32,0.0029909332593282064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,64,0.014781866470972696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,7168,32,0.01460906664530436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,12288,0.036209066708882645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,8192,0.025857067108154295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,65536,0.14369813601175946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,65536,0.15687893231709799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,65536,0.28131306966145836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,51200,0.22268800735473632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,16384,0.0756661335627238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,51200,0.12731093565622967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,16384,0.05798186858495077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,51200,0.11406506697336834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,12288,0.05790613492329916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,16384,0.04633173147837321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,5120,0.018104533354441323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,12288,0.032433066765467325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,12288,0.042641067504882814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,8192,0.03956693410873413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,10240,0.0377781351407369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,10240,0.027801599105199176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,8192,0.0339626669883728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,7168,0.03447573184967041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,7168,0.031651200850804646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,6144,0.0304639995098114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,6144,0.02922240098317464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,5120,0.025614933172861738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,5120,0.02678826649983724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,5120,0.01641813317934672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,4096,0.021386667092641195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,4096,0.024680533011754355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,4096,0.013982933759689332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,3584,0.019230933984120686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,3584,0.013119999567667642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,3584,0.023180800676345825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,3072,0.016923733552296958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,3072,0.02207466761271159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,2560,0.014595199624697366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,1536,0.009170132875442504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,2560,0.020722132921218873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,10240,0.0498741348584493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,2048,0.012414933244387309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,8192,0.023082667589187623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,2048,0.019662932554880778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,1536,0.009987200299898785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,6144,0.01879146695137024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,7168,512,0.005621333420276642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,1536,0.01839359998703003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,1024,0.007494399944941203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,1536,0.008646399776140849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,768,0.006410666803518932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,1024,0.015252266327540079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,768,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,512,0.0038293334345022834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,512,0.015094400445620219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,7168,0.02109760046005249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,256,0.003433600068092346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,256,0.014948266744613647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,3072,0.011688533425331115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,128,0.0031040000418821974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,128,0.014717866977055868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,64,0.0029290666182835894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,64,0.014819199840227762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,6144,32,0.002918400118748347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,6144,32,0.014749866724014283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,2560,0.011005866527557372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,65536,0.2400277296702067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,65536,0.1379861354827881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,2048,0.009486933549245197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,1024,0.006243200103441874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,768,0.005679999788602194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,51200,0.10668480396270752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,51200,0.09929493268330893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,51200,0.18982079823811848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,16384,0.06283200184504191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,16384,0.03548479874928792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,16384,0.04639893372853597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,12288,0.05325226783752442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,512,0.0054400001962979635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,12288,0.03738773266474406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,10240,0.04123733441034953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,8192,0.033581864833831784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,10240,0.024869332710901894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,10240,0.033632000287373856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,8192,0.030139732360839843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,7168,0.028177066644032793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,7168,0.02821333408355713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,6144,0.024920533100763954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,6144,0.02622186740239461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,5120,0.02109439969062805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,5120,0.024281599124272666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,5120,0.014651733636856078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,4096,0.017655466000239053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,4096,0.022547199328740438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,256,0.004993066688378652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,3584,0.01590506633122762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,3584,0.02195840080579122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,3584,0.012116266290346782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,3072,0.014065066973368326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,3072,0.020567466815312706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,6144,128,0.004782933493455251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,2560,0.012185600399971009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,2560,0.019719467560450236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,2560,0.010331733028093974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,2048,0.010442666212717692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,2048,0.018651733795801796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,2048,0.008987733721733093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,1536,0.008713600039482117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,1536,0.016992000738779704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,1536,0.00783679982026418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,1024,0.006809600194295247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,1024,0.015569067001342774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,1024,0.006213333209355672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,768,0.005318399767080943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,65536,0.12487680117289227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,768,0.015224533279736838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,768,0.005748266478379568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,512,0.0036746665835380556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,512,0.005452799797058106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,512,0.01509119967619578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,12288,0.02871893246968587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,256,0.0032543999453385672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,256,0.014773333072662353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,128,0.0029877332349618276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,8192,0.02062186598777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,128,0.014707199732462563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,64,0.002809600035349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,7168,0.01876373291015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,5120,32,0.0028864001234372456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,6144,0.016850133736928306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,64,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,5120,32,0.01451520025730133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,65536,0.1067466656366984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,65536,0.1852490743001302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,51200,0.14723946253458658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,16384,0.05015039841334025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,4096,0.01285546620686849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,51200,0.08898133436838786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,16384,0.039130667845408126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,12288,0.039230934778849286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,12288,0.03382186492284139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,12288,0.024951465924580894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,10240,0.0333077331384023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,10240,0.03031040032704671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,10240,0.021505065759023032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,8192,0.027480532725652058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,8192,0.02699093421300252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,256,0.005172266562779745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,7168,0.024949334065119424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,7168,0.02531840006510417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,7168,0.016742400328318276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,6144,0.02213653326034546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,6144,0.02408533294995626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,6144,0.015082666277885437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,3072,0.010923733313878376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,65536,0.10924800237019856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,5120,0.01930026610692342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,5120,0.02246613303820292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,4096,0.015915733575820924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,4096,0.020973867177963255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,4096,0.011636267105738323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,3584,0.014401066303253173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,3584,0.02018346587816874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,3072,0.01195199986298879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,3072,0.01951573292414347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,8192,0.0179967999458313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,2560,0.01053439974784851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,2560,0.018668800592422485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,2560,0.009657599528630574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,2048,0.009026133020718892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,2048,0.017638399203618368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,2048,0.008211199939250947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,1536,0.007714133461316426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,1536,0.0151637335618337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,3584,0.011108266313870747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,1024,0.006060799956321717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,1024,0.015431466698646545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,1024,0.005966933568318685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,51200,0.08561813036600749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,768,0.015171200037002563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,16384,0.03127893408139547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,768,0.005513600011666616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,512,0.0035402665535608927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,512,0.005339733262856802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,256,0.0031615999837716425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,512,0.015052800377209982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,256,0.004972800115744273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,128,0.0029290666182835894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,256,0.014699733257293702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,128,0.004745600124200186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,64,0.0028064000109831494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,128,0.014761599898338317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,32,0.002890666574239731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,64,0.014600533246994018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,5120,128,0.004820266862710317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,4096,32,0.014702933033307395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,65536,0.1693994681040446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,65536,0.10194346904754639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,51200,0.13380373318990071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,65536,0.10322453180948894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,16384,0.04790613253911336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,51200,0.0840394655863444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,12288,0.03456746737162272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,16384,0.029978666702906293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,12288,0.02406719923019409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,10240,0.029811199506123858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,10240,0.0288266658782959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,5120,0.01317759950955709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,4096,768,0.003933866570393244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,8192,0.024708267052968344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,8192,0.01765120029449463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,7168,0.022504534324010214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,7168,0.024152533213297526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,3072,0.010163199901580811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,7168,0.016309332847595216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,6144,0.020139733950297035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,6144,0.023178666830062866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,5120,0.01730666756629944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,5120,0.021667200326919555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,16384,0.03628586530685425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,4096,0.014641066392262777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,4096,0.02038080096244812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,12288,0.03197759985923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,4096,0.011490133404731751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,3584,0.013051733374595642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,3584,0.010898133118947346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,3072,0.01092693308989207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,3072,0.01919893423716227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,3072,0.009874133268992107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,2560,0.009629866480827332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,2560,0.01866133411725362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,4096,1536,0.006670933465162914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,8192,0.02581973274548848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,2560,0.00943893293539683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,2048,0.008578133583068848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,1536,0.00716480016708374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,2048,0.0170197327931722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,1536,0.0067104001839955645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,1536,0.015336533387502035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,1024,0.00491946687301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,1024,0.015124266346295675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,3584,0.019713066021601357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,768,0.004069333275159201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,51200,0.08161493142445883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,768,0.01504746675491333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,512,0.003618133316437403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,768,0.005589333176612854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,512,0.014751999576886495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,256,0.0032032000521818793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,256,0.014523733655611673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,10240,0.020946133136749267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,128,0.0029056000212828318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,128,0.014441600441932679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,6144,0.014710399508476257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,64,0.0028181334336598715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,64,0.014475733041763306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3584,32,0.0028245332340399425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,5120,0.013133866588274637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3584,32,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,65536,0.0887221336364746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,65536,0.1448479970296224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,65536,0.09468479951222739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,51200,0.1143338680267334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,51200,0.07187946637471518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,16384,0.03988266785939534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,16384,0.03470293283462524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,16384,0.02788693308830261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,12288,0.03142293294270833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,2048,0.0077802668015162155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,12288,0.029618134101231892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,10240,0.026719999313354493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,12288,0.02231360077857971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,10240,0.01958400011062622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,10240,0.027012266715367633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,8192,0.022997333606084188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,8192,0.02476480007171631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,1024,0.005774933099746704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,7168,0.020480000972747804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,6144,0.01821333368619283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,7168,0.014595199624697366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,7168,0.023846399784088135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,6144,0.022290132443110146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,6144,0.013389866550763449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,5120,0.016023466984430947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,4096,0.013889066378275552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,5120,0.021062399943669638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,4096,0.020077866315841675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,3584,0.01263146698474884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,3584,0.019259732961654664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,3584,0.010529067118962605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,3072,0.01144426663716634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,3072,0.01862186590830485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,3072,0.009505066275596618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,2560,0.008957866827646892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,2560,0.017862399419148765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,512,0.005435733497142792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,2560,0.008764800429344178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,2048,0.007816533247629803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,2048,0.015958399573961893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,1536,0.006414933502674103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,256,0.00508480022350947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,1536,0.015894400080045064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,1536,0.006634666522343953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,1024,0.004699733356634776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3584,128,0.00477866679430008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,1024,0.015334399541219077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,768,0.004109866668780645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,768,0.015107199549674988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,512,0.0035957333942254386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,512,0.014845866958300272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,51200,0.07520639896392822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,2048,0.00730453332265218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,256,0.003139200061559677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,256,0.01469546655813853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,128,0.0033546666304270422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,128,0.014551466703414917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,128,0.004755199948946635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,64,0.0028362666567166646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,64,0.014517333110173544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,3072,32,0.0028607999285062153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,3072,32,0.014600533246994018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,8192,0.016482133666674295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,65536,0.12060480117797852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,65536,0.07678506374359131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,5120,0.012123733758926392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,65536,0.09274880091349283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,4096,0.011006933450698853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,51200,0.06156053145726522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,16384,0.033427198727925614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,51200,0.07363093694051107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,12288,0.026434133450190227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,16384,0.027344000339508057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,16384,0.03056959907213847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,10240,0.022805333137512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,12288,0.02193386753400167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,12288,0.026740266879399614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,8192,0.019287467002868652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,10240,0.02484906713167826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,8192,0.022990934054056802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,1024,0.0059445331494013464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,7168,0.017473065853118898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,768,0.00557226687669754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,7168,0.022061866521835328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,512,0.005273599922657013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,6144,0.015404799580574035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,51200,0.09582186539967855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,5120,0.013751467068990072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,6144,0.020947200059890748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,5120,0.020308266083399452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,4096,0.011806933085123698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,4096,0.019222400585810342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,3072,256,0.005043200155099233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,3584,0.010999466975529988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,3584,0.01873706579208374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,3584,0.010327466328938802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,3072,0.010163199901580811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,3072,0.01727786660194397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,7168,0.015128533045450846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,2560,0.0082997332016627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,2560,0.01644480029741923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,2560,0.008188800017038981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,2048,0.007237333556016285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,2048,0.016365866859753928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,2048,0.007230933507283528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,1536,0.0056309332450230915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,1536,0.015735466281572977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,1536,0.006546133259932201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,1024,0.004439466694990794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,10240,0.019246933857599895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,1024,0.015312000115712484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,1024,0.0058677335580190025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,768,0.004023466755946477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,768,0.005544533332188925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,512,0.0035648000737031303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,8192,0.01623466710249583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,512,0.014998400211334228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,256,0.0030805334448814393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,512,0.005246933301289876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,128,0.002959999938805898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,128,0.01460693379243215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,6144,0.013614933689435324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,64,0.0027583998938401537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,5120,0.01229759951432546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,64,0.014641066392262777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2560,32,0.0027434666951497394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,32,0.014595199624697366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,4096,0.010857599973678588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,65536,0.09625813166300455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,65536,0.0636576016743978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,65536,0.0851477305094401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,51200,0.0754197359085083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,768,0.0151829332113266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,16384,0.02758293350537618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,51200,0.05293546517690023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,16384,0.027841067314147948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,51200,0.06776106357574463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,16384,0.025212800502777098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,12288,0.021922133366266885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,12288,0.020388267437616982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,10240,0.01908479928970337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,12288,0.024213333924611412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2560,256,0.014614400267601014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,10240,0.02290346622467041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,8192,0.01598186691602071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,10240,0.017854932943979898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,8192,0.021591466665267945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,7168,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,7168,0.01421333352724711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,7168,0.020696532726287842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,6144,0.013037866353988648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,6144,0.020294400056203206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,5120,0.01165226697921753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,4096,0.01036906639734904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,4096,0.018580265839894614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,3072,0.008974933624267578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,3584,0.00959999958674113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,3584,0.017041067282358803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,3584,0.009516800443331402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,3072,0.009032533566157023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,3072,0.016615466276804606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,3072,0.008556800087292989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,2560,0.0076682666937510175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,2560,0.016315733393033348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,2560,0.007803733150164287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,2048,0.006730666756629944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,2048,0.01585599978764852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,2048,0.007047466437021891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,1536,0.004724266628424326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,5120,0.011798399686813354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,1536,0.0156960000594457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,1024,0.004057600100835165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,1536,0.00652159998814265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,1024,0.00565119981765747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,1024,0.015267200271288552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,768,0.003701333453257879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,768,0.015200000007947287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,256,0.004955733319123586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,512,0.003398400048414866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,512,0.014595199624697366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2560,128,0.004862933357556661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,256,0.0030794667700926462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,512,0.005218133330345154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,256,0.004835199813048045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,256,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,128,0.002811733384927114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,128,0.004694400231043497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,64,0.002734933296839396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,64,0.01427733302116394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,2048,32,0.0026880001028378804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,32,0.014358400305112203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,65536,0.0744821310043335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,65536,0.052767999966939295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,5120,0.0197760005791982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,51200,0.059417601426442465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,768,0.005437866846720377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,8192,0.015379200379053751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,51200,0.04489920139312744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,6144,0.01300159990787506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,16384,0.022245333592096964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,16384,0.02472533384958903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,16384,0.02441493272781372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,12288,0.01810773412386576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,12288,0.019925334056218467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,12288,0.022851200898488362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,10240,0.01565226713816325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,8192,0.013291733463605246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,10240,0.021156267325083414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,2048,128,0.014525866508483887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,8192,0.020060799519220986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,2048,4096,0.010115200281143188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,7168,0.019690666596094766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,6144,0.011267200112342834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,6144,0.019063466787338258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,6144,0.012711466352144877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,5120,0.010246400038401287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,5120,0.01844053268432617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,7168,0.012292266885439555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,4096,0.009091200431187947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,4096,0.017061332861582436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,4096,0.009559466441472372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,3584,0.008641067147254943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,3584,0.017298134167989095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,3072,0.008214400211970011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,3072,0.016827734311421712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,65536,0.08419946829477945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,2560,0.00662720004717509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,2560,0.016215466459592185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,2560,0.007677866518497467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,2048,0.0055754666527112326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,2048,0.015758933623631795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,2048,0.006773333251476288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,1536,0.004779733220736186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,1536,0.015530666708946228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,51200,0.06688960393269858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,1024,0.004041599979003271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,1536,0.006542933483918508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,1024,0.005750399827957153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,1024,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,10240,0.017448532581329345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,768,0.003704533229271571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,512,0.0033632000287373864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,768,0.015084800124168397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,8192,0.015106133619944253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,256,0.0030400000512599947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,512,0.005087999999523163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,7168,0.013890133301417033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,512,0.014935466647148132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,256,0.00490880012512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,128,0.0028757333755493166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,256,0.0144896000623703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,128,0.014405333002408347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,64,0.002757333219051361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1536,32,0.0027637332677841187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,64,0.014385066429773965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1536,32,0.01444586714108785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,65536,0.050348798433939614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,65536,0.04057173331578572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,65536,0.08402559757232667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,51200,0.03503893216451009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,5120,0.010980266332626342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,51200,0.06661866505940756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,16384,0.021870932976404824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,16384,0.02479893366495768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,12288,0.013347199559211731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,12288,0.01989333430926005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,10240,0.012055466572443645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,10240,0.019850667317708334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,10240,0.017486933867136636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,8192,0.010398933291435241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,8192,0.019105066855748497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,8192,0.014844800035158793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,7168,0.009851732850074768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,7168,0.018169599771499633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,768,0.005347200234731038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,7168,0.013401599725087485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,6144,0.008801066875457763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,6144,0.017860267559687296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,5120,0.008441600203514098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,3584,0.008922666311264038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,5120,0.017990400393803917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,51200,0.040423464775085446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,5120,0.010570666193962098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,16384,0.016040533781051636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,3072,0.00813973347345988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,4096,0.017299199104309083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,4096,0.009365333120028178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,3584,0.017051732540130614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,3584,0.008878933389981587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,3072,0.00647573322057724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,3072,0.016520532965660095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,2560,0.0060703997810681665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,12288,0.01996586720148722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,2560,0.016204800208409628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,2048,0.005385600030422211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,2048,0.01583146651585897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,4096,0.007928533355395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1536,128,0.004699733356634776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,1536,0.004694400231043497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,3584,0.0070613334576288865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,1536,0.015518933534622192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,1536,0.00644053320089976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,1024,0.005795200169086456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,1024,0.01532266636689504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,768,0.003681066632270813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,768,0.015092266599337259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,768,0.005369600156943003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,512,0.014812800288200378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,6144,0.011782399813334147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,256,0.0030048000315825146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,256,0.01458026667435964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,128,0.002902399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,1024,0.004053333401679992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,128,0.014416000247001648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,128,0.004756266872088114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,64,0.0026591998835404714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,32,0.0026709333062171934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,64,0.014594133694966635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,1024,512,0.0034346667428811393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,65536,0.039562666416168214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,1024,32,0.014593066771825156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,51200,0.03219946622848511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,65536,0.036152533690134686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,65536,0.08438826402028402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,16384,0.013479466239611307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,51200,0.06649279991785685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,3072,0.008077866832415263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,16384,0.0205567995707194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,16384,0.024703999360402427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,12288,0.01146346628665924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,10240,0.010386133193969726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,12288,0.019799466927846274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,12288,0.019688532749811808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,2560,0.007698133091131846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,8192,0.009410132964452107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,10240,0.017262933651606242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,10240,0.018731733163197838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,7168,0.008586666981379191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,8192,0.014443733294804893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,8192,0.01754986643791199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,7168,0.017990400393803917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,7168,0.013172266880671182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,5120,0.007301333546638489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,6144,0.011735467116038005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,6144,0.01745706597963969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,5120,0.01844480037689209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,4096,0.0063733334342638654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,4096,0.009246933460235595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,4096,0.01731626590092977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,3584,0.006666666766007741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,3584,0.016911999384562174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,3072,0.006093866626421611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,3072,0.01684373418490092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,51200,0.031199999650319415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,2560,0.006126933296521505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,2560,0.016330666343371072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,2560,0.007743999858697255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,2048,0.005416533350944519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,2048,0.015938133001327515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,512,0.005129600067933401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,6144,0.00830080012480418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,2048,0.0068010667959849044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,1536,0.0047423998514811196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,1024,0.004070399949947992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,1024,0.015049599607785544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,1024,0.005755733450253805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,768,0.0036202666660149893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,768,0.01493119994799296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,768,0.005379199981689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,512,0.0033791999022165934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,512,0.014777599771817525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,1024,256,0.004790399968624115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,256,0.0030805334448814393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,256,0.014500266313552857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,256,0.004877866804599762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,128,0.0028512001037597655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,128,0.014441600441932679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,128,0.004710400104522705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,64,0.0026410666604836782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,1536,0.015585066874821982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,64,0.014703999956448874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,768,32,0.002755200117826462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,768,32,0.014658133188883463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,65536,0.02792853315671285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,5120,0.010553600390752156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,65536,0.02865920066833496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,51200,0.023013333479563393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,51200,0.025922133525212603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,51200,0.06642346779505412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,3584,0.008975999553998311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,16384,0.010554666320482891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,16384,0.01897066632906596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,16384,0.024370133876800537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,3072,0.008147199948628742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,12288,0.009250133236249288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,12288,0.01763733426729838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,12288,0.019451733430226645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,10240,0.008685866991678875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,8192,0.0075989335775375364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,10240,0.01686933239301046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,10240,0.018023467063903807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,7168,0.006796800096829732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,8192,0.017526400089263917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,7168,0.017989333470662436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,6144,0.006345599889755249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,6144,0.018102399508158364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,6144,0.011665067076683045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,5120,0.006365866462389629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,5120,0.018203733364741008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,1536,0.006512000163396199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,5120,0.010479999581972758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,4096,0.005941333373387655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,4096,0.0176362673441569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,4096,0.009265066186587016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,3584,0.006461866696675618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,3584,0.017067732413609822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,3072,0.006071466704209646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,2048,0.006820266445477803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,3072,0.016661333044370015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,2560,0.006208000083764395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,2560,0.016247466206550598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,2560,0.007578666508197785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,2048,0.00544106662273407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,2048,0.016013866662979125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,768,512,0.005115733544031779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,1536,0.004766933123270671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,1536,0.015595733126004537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,7168,0.012932266791661581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,1024,0.004204800228277842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,1024,0.015233066678047181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,768,0.0037151999771595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,65536,0.08351999918619792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,768,0.015035733580589294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,512,0.003385599950949351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,512,0.014820266763369241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,512,0.004996266464392344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,256,0.003110400090614955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,256,0.014628266294797262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,256,0.004737066725889841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,128,0.0028351999819278715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,8192,0.014150399963061014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,128,0.01446293294429779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,64,0.0026848000784715016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,64,0.014421332875887552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,512,32,0.0026762666801611584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,512,32,0.014526933431625366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,65536,0.01761066714922587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,65536,0.023248000939687093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,3584,0.008834133545557659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,51200,0.013894400000572205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,3072,0.008001066744327545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,51200,0.021552000443140665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,51200,0.06612266699473063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,16384,0.007810133198897044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,16384,0.018186666568120322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,12288,0.00639466643333435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,12288,0.017534933487574258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,2048,0.006853333115577698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,10240,0.006402133405208588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,10240,0.01763520042101542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,10240,0.016504533092180886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,8192,0.006343466540177663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,1536,0.006363733112812043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,8192,0.017435733477274576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,7168,0.006182399888833364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,7168,0.017849600315093993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,1024,0.005755733450253805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,6144,0.005940266450246175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,6144,0.017539199193318686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,6144,0.01167146662871043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,5120,0.006284800171852112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,5120,0.017960532506306966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,5120,0.010488532980283101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,4096,0.005896533528963724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,4096,0.017272533973058064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,12288,0.018931200106938682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,3584,0.00633493314186732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,768,0.005381333331267038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,3584,0.017041067282358803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,3584,0.008777599533398945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,3072,0.006018133461475372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,3072,0.016455466548601784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,3072,0.007982933521270752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,2560,0.005957333246866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,2560,0.016549332936604818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,512,128,0.004637866715590159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,2048,0.005305600166320801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,2048,0.016235733032226564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,2048,0.0067552000284194945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,1536,0.004737066725889841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,1536,0.015894400080045064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,65536,0.08301440080006918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,1024,0.003955200066169103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,1024,0.015185067057609558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,1024,0.005570133527119955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,768,0.00363520011305809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,768,0.015051733454068503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,16384,0.023975465695063272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,512,0.0033589333295822145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,512,0.014934399724006652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,512,0.005016533533732096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,256,0.0029738667110602063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,256,0.014669866363207499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,256,0.004755199948946635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,128,0.0028607999285062153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,2560,0.007520000139872233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,128,0.014513066411018372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,64,0.0027189334233601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,64,0.01460906664530436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,8192,0.014179199934005737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,256,32,0.002690133452415466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,256,32,0.014401066303253173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,7168,0.01286186675230662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,65536,0.0206986665725708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,51200,0.010275200009346008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,65536,0.08313279946645101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,51200,0.018347734212875368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,51200,0.06585813363393148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,16384,0.005964800218741099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,16384,0.017602133750915527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,12288,0.005946666498978933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,12288,0.01762346625328064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,12288,0.018900267283121743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,10240,0.006296533346176148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,768,0.005302399893601735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,10240,0.01763946612675985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,8192,0.006206933160622915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,4096,0.009178666273752849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,8192,0.017434666554133095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,7168,0.005971199770768484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,7168,0.017429333925247193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,6144,0.005781333148479462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,65536,0.01188693344593048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,6144,0.017542399962743125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,5120,0.006146133442719777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,5120,0.017604267597198485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,5120,0.010411733388900756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,4096,0.005810133119424184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,4096,0.01698026657104492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,1536,0.006437333424886067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,3584,0.006182399888833364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,3584,0.016742400328318276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,6144,0.011653332908948263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,3072,0.00586346685886383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,3072,0.016406400005022685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,3072,0.007889066636562348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,256,128,0.0046965335806210835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,2560,0.005785599847634633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,2560,0.007566933333873749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,2560,0.01603626708189646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,2048,0.005092266698678335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,2048,0.015701333681742348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,2048,0.006704000135262807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,1536,0.004499199986457825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,1536,0.015448533495267234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,1024,0.003920000046491623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,1024,0.014986667037010192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,16384,0.023781333367029825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,768,0.003568000098069509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,1024,0.005619200070699056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,768,0.005235200126965841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,768,0.01478506624698639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,512,0.0032992000381151833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,512,0.014573867122332254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,256,0.002996266633272171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,256,0.014296533664067588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,10240,0.016354133685429893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,256,0.0048767998814582825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,128,0.002834133307139079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,8192,0.01411733329296112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,128,0.014180266857147216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,64,0.002681600054105123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,128,32,0.002699733277161916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,64,0.014241066575050355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,128,32,0.014358400305112203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,7168,0.012902399897575379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,51200,0.007467733323574066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,65536,0.019013333320617675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,16384,0.0059445331494013464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,51200,0.018729599316914876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,16384,0.01746986707051595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,12288,0.005981866518656413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,12288,0.01723626653353373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,10240,0.005960533519585928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,10240,0.017748266458511353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,8192,0.005899733304977417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,8192,0.017560533682505288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,7168,0.005795200169086456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,7168,0.017513600985209148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,4096,0.009141332904497783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,6144,0.005710933109124502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,5120,0.0060127998391787205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,6144,0.01729066570599874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,4096,0.005560533205668131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,5120,0.017694934209187826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,3584,0.006097066899140676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,4096,0.01706026593844096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,3072,0.005712000032265982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,3584,0.0088128000497818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,3584,0.016718933979670204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,2560,0.0056650668382644655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,3072,0.016321067015329996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,2048,0.005054933329423269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,2560,0.015870933731396995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,1536,0.004481066763401031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,2048,0.01567893326282501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,1024,0.00388373335202535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,1536,0.015304533640543619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,768,0.003570133447647095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,65536,0.008407466610272725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,1024,0.014967466394106546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,512,0.00325546662012736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,768,0.014846932888031007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,128,0.0027647999425729113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,256,0.014509866635004679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,64,0.0026591998835404714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,128,0.014403200149536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,32,0.002624000112215678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,64,0.014492799838383993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,65536,0.00876586635907491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,32,0.014338133732477823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,51200,0.008014933268229166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,65536,0.019065600633621217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,16384,0.005851733187834421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,51200,0.018742400407791137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,12288,0.005957333246866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,16384,0.017541333039601644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,10240,0.005994666616121928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,12288,0.01707306702931722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,8192,0.005886933207511902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,10240,0.017678932348887125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,7168,0.005691733459631602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,8192,0.017059199015299478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,6144,0.005640533566474914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,7168,0.017968000968297322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,5120,0.0059232001503308615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,6144,0.017083734273910522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,4096,0.005582933127880096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,5120,0.017883733908335368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,3584,0.005983999868233999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,4096,0.016812799374262492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,3072,0.005748266478379568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,3584,0.016746666034062704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,2560,0.005648000041643778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,1536,0.006423466900984447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,3072,0.01623146633307139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,2560,0.01600106656551361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,2048,0.01593280037244161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,1536,0.004477866490681966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,1024,0.0037941334148248037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,128,0.004661333560943603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,1536,0.015478400389353433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,768,0.00345920001467069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,9,128,512,0.0050240000089009605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,1024,0.01492800017197927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,512,0.0031839999059836066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,256,0.0028768000503381092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,512,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,256,0.01448319951693217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,64,256,0.0029418667157491045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,64,0.002583466718594233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,64,512,0.014504533012708029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,128,0.014248533050219217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,32,0.002616533388694127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,64,0.014243200421333313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,32,0.014167466759681701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,2048,0.005062399804592133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,16384,0.7545109430948893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,9,32,768,0.015029333035151162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,9,32,128,0.0026986666023731233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,16384,0.3911295890808105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,16384,0.35606400171915686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,12288,0.29757121404012044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,12288,0.5701855977376302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,51200,1.1809141794840494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,10240,0.2494581381479899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,10240,0.47318401336669924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,10240,0.22450772921244302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,8192,0.38298559188842773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,7168,0.1800725301106771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,8192,0.18037546475728353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,8192,0.23851733207702636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,7168,0.3297055880228678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,51200,2.33797124226888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,7168,0.1590122699737549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,6144,0.1569119930267334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,6144,0.28481388092041016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,5120,0.133188263575236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,4096,0.19408960342407228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,5120,0.2794346809387207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,4096,0.10949333508809407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,4096,0.10858773390452067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,3584,0.09847466945648194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,3584,0.19718507130940754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,3584,0.08323413530985514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,3072,0.14464853604634603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,3072,0.07202133337656656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,3072,0.09904853502909342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,2560,0.07482879956563314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,2560,0.06143893400828043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,2048,0.0969760020573934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,2048,0.06269866625467936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,1536,0.07449173132578532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,1536,0.05156053304672241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,12288,0.26811625162760416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,1536,0.03955519994099935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,51200,1.0915541330973306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,1024,0.04282666842142741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,1024,0.05061759948730469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,768,0.039579733212788897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,768,0.03363093137741089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,1024,0.03161919911702474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,512,0.027313067515691118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,512,0.017145599921544394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,256,0.015200000007947287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,2560,0.12087360223134358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,512,0.02918933431307475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,256,0.0207914670308431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,256,0.012134400010108948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,128,0.01002239982287089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,128,0.009416533509890239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,64,0.006257066627343495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,32,0.004445866743723551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,128,0.017678932348887125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,64,0.016189866264661155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,32,0.01613759994506836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,6144,0.15287040074666342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,5120,0.13204800287882487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,2048,0.050600532690684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,51200,0.9617866516113281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,65536,1.2082655588785807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,51200,1.8568288167317708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,768,0.02288533250490824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,16384,0.6063882827758789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,16384,0.35226666132609047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,65536,2.3629493713378906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,12288,0.23745387395222983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,10240,0.3783978780110677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,12288,0.5296117464701335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,10240,0.23044266700744628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,8192,0.30768000284830727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,8192,0.1881696065266927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,7168,0.14452053705851237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,7168,0.26076693534851075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,6144,0.2259317398071289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,6144,0.1289184014002482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,6144,0.11488107045491536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,65536,1.1540074666341147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,5120,0.18760746320088703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,51200,0.9054250717163086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,5120,0.09687679608662923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,5120,0.12290346622467041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,4096,0.1753226598103841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,4096,0.08977386951446534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,3584,0.15355092684427898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,16384,0.34463361104329426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,3584,0.0801258643468221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,3584,0.06998613675435385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,3072,0.07139306863149007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,3072,0.060716799894968664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,3072,0.13283200263977052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,2560,0.06174720128377279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,2560,0.09677013556162516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,12288,0.24416319529215494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,2048,0.07742506663004557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,2048,0.052741332848866784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,10240,0.18672320048014324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,1536,0.058968532085418704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,1536,0.04336320161819458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,1536,0.03766506512959798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,1024,0.04059733152389526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,1024,0.034357333183288576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,768,0.03145920038223267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,768,0.02918720046679179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,512,0.02215253313382467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,512,0.024112000068028768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,4096,0.08969706694285075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,512,0.015247999628384908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,256,0.01953173279762268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,256,0.013513599832852682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,8192,0.15007786750793456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,256,0.010857599973678588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,128,0.015643733739852905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,128,0.00848533312479655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,128,0.007910400132338206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,64,0.01569919983545939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,32,0.004136533290147781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,51200,32,0.015896532932917276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,7168,0.13247146606445312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,65536,0.3782549222310384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,65536,0.7408416112263997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,51200,64,0.004342400034268697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,65536,0.35407466888427735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,51200,0.299460252126058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,51200,0.581707763671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,51200,0.2787274678548177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,2560,0.05221653381983439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,16384,0.18596906661987306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,16384,0.12424960136413574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,2048,0.047804800669352214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,16384,0.10991466840108235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,1024,0.02481493353843689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,12288,0.0848416010538737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,51200,768,0.020151466131210327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,10240,0.07283733685811361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,10240,0.11665066878000896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,8192,0.09510293006896972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,8192,0.06154026587804159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,8192,0.04984960158665975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,7168,0.04450240135192871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,7168,0.05572906732559204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,6144,0.05009706815083822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,6144,0.072269868850708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,6144,0.039019731680552165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,5120,0.0603061318397522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,5120,0.03342506488164266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,5120,0.04432106812795003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,4096,0.049116798241933185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,4096,0.03827093442281087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,4096,0.027720532814661664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,3584,0.04336959918340047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,3584,0.03573653300603231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,3072,0.03780693213144938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,3072,0.03300586740175883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,3584,0.02537600000699361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,2560,0.03214400013287862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,2560,0.029778132836023968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,2048,0.026126933097839356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,2048,0.026521599292755126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,2048,0.01649066706498464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,1536,0.020595200856526694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,1536,0.023465599616368613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,1536,0.013682132959365845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,1024,0.014620799819628397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,1024,0.020266666014989217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,1024,0.01076586643854777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,768,0.011594667037328085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,768,0.018990933895111084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,512,0.008666666348775227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,12288,0.14155519803365071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,512,0.017478400468826295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,512,0.007585066556930542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,256,0.005145599941412607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,256,0.005318399767080943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,256,0.014968533317248026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,128,0.003455999990304311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,128,0.01477120021979014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,128,0.004983466863632202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,64,0.0031328000128269195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,7168,0.08276693026224771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,32,0.0031498665610949195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,64,0.014667733510335287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,32,0.014908799529075622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,65536,0.2933984120686849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,65536,0.5697216033935547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,51200,0.43508265813191727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,12288,0.07213333447774252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,10240,0.060794667402903235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,16384,0.14409707387288412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,16384,0.08730346361796061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,16384,0.07551039854685465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,12288,0.11061013539632161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,12288,0.06881919701894125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,10240,0.09264533519744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,3072,0.02243306636810303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,10240,0.05856639941533407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,10240,0.04884159962336222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,2560,0.019827200969060262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,8192,0.07592960198720297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,8192,0.049635199705759685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,7168,0.06653439998626709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,8192,0.040411734580993654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,7168,0.03605973323186239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,768,0.009242666761080424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,51200,0.2367296059926351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,6144,0.05527360041936239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,6144,0.04143466552098592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,6144,0.031523199876149495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,5120,0.036638931433359785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,5120,0.027361067136128743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,5120,0.05262080033620199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,4096,0.036855467160542804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,4096,0.032459733883539836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,3584,0.03261973261833191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,3584,0.030273065964380903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,3584,0.021077332894007365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,3072,0.031143466631571453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,3072,0.028114134073257448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,3072,0.01876373291015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,2560,0.02448106606801351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,2560,0.025462400913238526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,2048,0.020433066288630168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,2048,0.023285333315531412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,2048,0.01376106639703115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,1536,0.015884799758593242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,1536,0.021310933430989585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,1536,0.011677866180737812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,1024,0.011734400192896526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,1024,0.019337600469589232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,1024,0.009196799993515015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,65536,0.283189328511556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,7168,0.045354668299357095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,768,0.009801600376764934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,512,0.007603199779987335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,768,0.008242133259773254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,768,0.019129600127538046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,512,0.005551999807357788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,512,0.01518933375676473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,256,0.0038581334054470064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,256,0.005201066533724466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,256,0.015093333522478738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,128,0.0034485332667827605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,64,0.003180799881617228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,128,0.0048426667849222815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,128,0.014924800395965577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,32,0.0032106667757034303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,64,0.014838400483131408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,32,0.015175466736157736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,65536,0.2531328042348226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,51200,0.22502506573994957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,65536,0.47711680730183914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,12288,0.057657599449157715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,51200,0.20206826527913413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,51200,0.3747584025065104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,51200,0.19705066680908204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,16384,0.12435413201649984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,16384,0.07653760115305583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,12288,0.05925120115280151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,16384,0.07396266460418702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,12288,0.09601173400878907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,10240,0.0787829319636027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,10240,0.05179093281428019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,8192,0.06495893398920695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,8192,0.04441279967625936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,8192,0.03482986688613891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,4096,0.022689066330591836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,7168,0.05357546806335449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,7168,0.04081706603368123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,7168,0.030918399492899578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,6144,0.04887466828028361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,6144,0.03739840189615885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,65536,0.2369333267211914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,5120,0.023788799842198692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,5120,0.03933653434117635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,4096,0.020496000846227012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,4096,0.03649919827779134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,2560,0.01662720044453939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,3584,0.02844160000483195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,3584,0.018637865781784058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,3584,0.02781333327293396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,3072,0.016819200913111367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,3072,0.02577280004819234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,2560,0.021331199010213218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,2560,0.023780266443888344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,2048,0.01737706661224365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,2048,0.021884800990422566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,2048,0.012546133001645407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,1536,0.013668266932169595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,1536,0.020346667369206747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,5120,0.034179198741912845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,1024,0.010388267040252686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,4096,0.029687466224034627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,1024,0.008596266309420269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,768,0.008424533406893413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,3072,0.024958932399749757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,768,0.0171615997950236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,768,0.007054933408896129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,512,0.00662720004717509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,512,0.00565226674079895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,512,0.015187199910481772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,256,0.0035061334570248926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,256,0.01504746675491333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,128,0.0032799998919169106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,128,0.014709333578745524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,128,0.004934399823347728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,64,0.0029738667110602063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,64,0.014881066481272378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,32,0.002996266633272171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,32,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,12288,0.04942506551742554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,10240,0.04191360076268514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,1024,0.0186463991800944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,6144,0.027374933163324993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,65536,0.3742815971374512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,65536,0.20137599309285484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,65536,0.18388907114664715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,51200,0.16089812914530438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,16384,0.09268799622853598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,51200,0.2955157279968262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,51200,0.14550933837890626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,2560,0.01461120049158732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,16384,0.05298453172047933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,16384,0.06062186559041342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,12288,0.03991466760635376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,12288,0.08278079827626547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,12288,0.0497162659962972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,10240,0.059432534376780186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,8192,0.04797226587931315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,10240,0.04380906820297241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,8192,0.03921066522598267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,7168,0.04231893221537272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,8192,0.02837120095888774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,7168,0.035393067200978595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,1536,0.010738133390744527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,6144,0.03707520167032878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,5120,0.031411200761795044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,6144,0.02286400000254313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,5120,0.0295413335164388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,5120,0.020152533054351808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,4096,0.025685334205627443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,4096,0.01710933248202006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,3584,0.022954666614532472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,3584,0.01577279965082804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,3584,0.024996266762415568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,3072,0.020205867290496827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,3072,0.02323946754137675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,3072,0.013755733768145243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,2560,0.017410133282343546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,2560,0.02204906741778056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,2048,0.014275200168291726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,2048,0.020525866746902467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,256,0.005160533388455709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,1536,0.011546666423479717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,7168,0.02576533357302348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,1536,0.01925440033276876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,6144,0.032585599025090534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,1024,0.008654933174451191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,1536,0.00977066655953725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,768,0.007303466896216075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,1024,0.007593599955240886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,512,0.0054848000407218935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,768,0.00617386649052302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,768,0.01506239970525106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,4096,0.026282666126887004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,512,0.005544533332188925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,256,0.003390933324893316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,512,0.0151936004559199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,256,0.014719999829928079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,128,0.003110400090614955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,128,0.004853333532810211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,128,0.01470080018043518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,64,0.014587733149528503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,32,0.014637866616249084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,65536,0.32751681009928385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,65536,0.1840490659077962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,10240,0.03422293265660604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,1024,0.01749546726544698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,64,0.0029663999875386557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,65536,0.163863468170166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,32,0.0029109333952267963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,51200,0.14908159573872884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,51200,0.2590218702952067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,16384,0.09036693572998047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,16384,0.04547946850458781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,16384,0.05788480043411255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,12288,0.07650026480356852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,12288,0.03623040119806926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,12288,0.04701866706212361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,2560,0.012587733070055642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,10240,0.056840535004933676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,8192,0.04493546485900879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,10240,0.04179733196894328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,10240,0.030730666716893514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,8192,0.03562239805857341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,7168,0.0401525338490804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,7168,0.02327466607093811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,7168,0.03344106674194336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,6144,0.03542506694793701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,6144,0.020914133389790854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,6144,0.030402133862177532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,5120,0.030308266480763752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,5120,0.027990400791168213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,4096,0.023155200481414794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,2048,0.011193600296974183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,4096,0.015316266814867655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,4096,0.02523626685142517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,3072,0.01843520005544027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,3584,0.014247467120488485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,3072,0.022699733575185142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,256,0.0050687998533248905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,2560,0.015818666418393454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,2560,0.02151573300361633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,8192,0.025643734137217204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,2048,0.012986666957537332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,2048,0.019961599508921304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,5120,0.01813653310139974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,3584,0.020820266008377074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,1536,0.010733866691589355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,3584,0.024238934119542442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,1536,0.018516266345977785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,1024,0.008187733093897502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,1024,0.016176000237464905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,1024,0.0066538666685422255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,768,0.006771199901898702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,768,0.015187199910481772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,768,0.005942399799823761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,512,0.003949866692225138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,512,0.005692799886067709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,512,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,256,0.003352533280849457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,256,0.014807466665903726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,128,0.003035733352104823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,128,0.014755200346310934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,128,0.00477866679430008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,64,0.0028704000016053517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,64,0.014716800053914389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,32,0.0029109333952267963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,32,0.014638933539390563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,51200,0.12985386848449706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,65536,0.2809770584106445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,2048,0.010152533650398254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,65536,0.15769599278767904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,65536,0.14365545908610028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,51200,0.2238154729207357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,51200,0.1264234701792399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,3072,0.012731732924779257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,16384,0.07530773480733235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,16384,0.0516543984413147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,51200,0.11415146986643473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,12288,0.05779733260472616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,12288,0.04285973310470581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,12288,0.03211306730906169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,10240,0.03805866638819377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,8192,0.03898986577987671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,10240,0.049480533599853514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,8192,0.033752532800038655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,7168,0.03407253424326579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,7168,0.03145706653594971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,6144,0.02994453310966492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,6144,0.029335467020670573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,6144,0.018910932540893554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,5120,0.025386667251586913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,5120,0.026242132981618243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,5120,0.01627626617749532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,4096,0.021373866001764934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,2560,0.01183893382549286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,4096,0.023918932676315306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,4096,0.014006400108337402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,3584,0.019118932882944743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,3584,0.0232586661974589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,3072,0.016826667388280234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,1536,0.009175466497739156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,3072,0.021766400337219237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,3072,0.011713066697120666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,2560,0.014626133441925048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,2560,0.020728532473246256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,2560,0.011047466595967611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,2048,0.012384000420570373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,2048,0.019388800859451293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,1536,0.010041600465774536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,1536,0.018232532342274985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,1536,0.008950400352478027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,1024,0.007563733557860057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,256,0.0051701332132021586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,1024,0.015299200018246969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,1024,0.0063274666666984555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,768,0.015239466230074564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,3584,0.012916266918182373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,512,0.00363520011305809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,512,0.014851199587186179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,512,0.005257600049177805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,256,0.0032640000184377036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,16384,0.04056426684061686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,256,0.014617600043614707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,128,0.0029578665892283124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,256,0.005037866532802582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,128,0.004774400095144907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,128,0.014426666498184203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,64,0.0028362666567166646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,32,0.0028319999575614927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,64,0.014360533157984415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,32,0.014664533734321594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,768,0.0064640000462532045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,10240,0.02759893337885539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,65536,0.13758187294006347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,65536,0.23948052724202476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,8192,0.022977066040039063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,51200,0.10840960343678792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,65536,0.12435093720753987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,7168,0.02102933327356974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,16384,0.06355839967727661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,51200,0.18898132642110188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,16384,0.04533119996388753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,12288,0.04883626699447632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,12288,0.0397269328435262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,10240,0.041861331462860106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,10240,0.024612265825271606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,10240,0.03354026476542155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,8192,0.03378453254699707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,8192,0.0300053338209788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,7168,0.028061866760253906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,7168,0.02831146717071533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,7168,0.018905599912007652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,6144,0.02473919987678528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,6144,0.026264532407124834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,6144,0.01694186727205912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,5120,0.021281067530314127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,2048,0.009764267007509868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,5120,0.02419840097427368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,4096,0.017550933361053466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,4096,0.022243199745814006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,4096,0.01270080010096232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,3584,0.015780267119407655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,3584,0.02158400019009908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,3584,0.012055466572443645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,3072,0.01395840048789978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,3072,0.02058239976565043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,768,0.005946666498978933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,2560,0.012196266651153564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,2560,0.01973653237024943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,2560,0.010321066776911417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,2048,0.01032533347606659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,2048,0.018730666240056357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,51200,0.09874026775360108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,16384,0.03544106483459473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,1536,0.009278933207194012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,12288,0.028572799762090047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,1536,0.017153066396713258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,1024,0.006830933193365733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,1024,0.006270933151245117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,1024,0.015255467096964518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,768,0.005382399757703145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,768,0.005702400207519531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,768,0.015170133113861084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,512,0.003522133330504099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,8192,0.020488532384236653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,256,0.0031498665610949195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,512,0.015050666530927024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,128,0.0029738667110602063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,256,0.005144533514976501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,256,0.014780799547831217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,64,0.002773333340883255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,128,0.0048320000370343525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,128,0.01474453310171763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,32,0.002796799937884013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,64,0.014754133423169455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,32,0.014674133062362671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,65536,0.10954986413319905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,65536,0.18696853319803874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,5120,0.014631467064221701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,51200,0.08916373252868652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,51200,0.1480533281962077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,51200,0.08526399930318197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,16384,0.0499018669128418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,16384,0.039018666744232176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,16384,0.03113493323326111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,12288,0.03915626605351766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,12288,0.033321599165598556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,10240,0.0333568016688029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,12288,0.025018666187922162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,10240,0.021604265769322714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,8192,0.02728319962819417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,3072,0.010897066195805867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,8192,0.02707306742668152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,7168,0.02469866673151652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,7168,0.025326933463414508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,7168,0.016790399948755898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,6144,0.02199466625849406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,6144,0.023945599794387817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,5120,0.01917866667111715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,5120,0.022405334313710532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,5120,0.013229866822560629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,4096,0.015867732961972556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,2048,0.008918399612108868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,4096,0.02081386645634969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,3584,0.013184000054995218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,4096,0.01183786690235138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,3584,0.011127466956774395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,3584,0.020386133591334024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,3072,0.011829333504041036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,3072,0.019760000705718993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,3072,0.010231467088063557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,2560,0.010321066776911417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,2560,0.01883946657180786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,2048,0.008937600255012512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,2048,0.017802667617797852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,10240,0.030585600932439165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,1536,0.007567999760309856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,1536,0.015340800086657206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,1536,0.007430399954319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,1536,0.006457599997520447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,1024,0.005993600189685822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,1024,0.015175466736157736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,512,0.0053493330876032506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,768,0.00391146664818128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,1024,0.005746133128801982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,768,0.0054954667886098225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,768,0.015109333395957946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,512,0.0034645333886146545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,512,0.014881066481272378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,256,0.003123199939727783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,256,0.014724266529083253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,65536,0.1069493293762207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,128,0.0029866665601730345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,128,0.014536533753077188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,128,0.004837333162625631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,64,0.002754133443037669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,64,0.014678399761517844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,32,0.0027413333455721537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,32,0.014651733636856078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,8192,0.018081067005793254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,65536,0.1689557393391927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,65536,0.10122559865315754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,6144,0.014922666549682616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,512,0.005187200009822845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,51200,0.1334506670633952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,51200,0.08361706733703614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,16384,0.04797333478927612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,51200,0.07828479607899984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,12288,0.03584106763203938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,16384,0.03644693295160929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,12288,0.023884799083073935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,12288,0.03143360018730164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,2560,0.00960106650988261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,10240,0.0300981342792511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,10240,0.02855573296546936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,8192,0.024868265787760416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,10240,0.020925867557525634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,8192,0.025509333610534667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,7168,0.02250666618347168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,2048,0.008289066453774769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,7168,0.024310400088628135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,6144,0.019875200589497884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,6144,0.023125332593917847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,6144,0.014588800072669984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,5120,0.017453867197036742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,5120,0.012698666254679362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,5120,0.02184106707572937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,4096,0.013317333658536276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,4096,0.020329600572586058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,4096,0.011105066537857056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,3584,0.01211199959119161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,3584,0.019654399156570433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,3584,0.011037866274515789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,3072,0.010889599720637005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,2560,0.009672533472379048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,3072,0.010129066308339436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,3072,0.019011199474334717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,2048,0.00842026670773824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,2560,0.018542933464050292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,2048,0.007272533575693766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,2048,0.015917866428693136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,1536,0.007169066866238911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,1536,0.015598932902018229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,1024,0.004621866842110952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,1024,0.015337600310643514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,1024,0.006060799956321717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,768,0.0038773333032925926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,768,0.015098667144775391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,256,0.004916266600290934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,512,0.003533866753180822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,512,0.014805333813031516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,1536,0.006500266492366791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,256,0.003142400085926056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,65536,0.10181866486867268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,256,0.014568533500035605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,128,0.0029386666913827257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,128,0.014376533031463624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,16384,0.029898667335510255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,64,0.002794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,64,0.014538666605949402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,32,0.002811733384927114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,32,0.014537599682807923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,7168,0.016115199526151022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,8192,0.017465599377950034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,65536,0.14426132837931316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,65536,0.08872319857279459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,65536,0.09449066321055094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,51200,0.07246613502502441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,51200,0.07513493696848551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,16384,0.039979732036590575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,16384,0.03349226713180542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,12288,0.03168746630350749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,2560,0.009278933207194012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,12288,0.02903146743774414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,12288,0.022394667069117226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,10240,0.02671146591504415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,10240,0.019449599583943687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,10240,0.026842667659123735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,8192,0.022249599297841392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,8192,0.024740266799926757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,8192,0.016556800405184428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,7168,0.019939200083414713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,768,0.0056202664971351625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,512,0.005363200108210246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,6144,0.017630932728449504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,51200,0.11402773062388102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,256,0.005008000135421753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,5120,0.015497600038846334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,5120,0.020895999670028687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,128,0.004830933113892873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,7168,0.023627734184265135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,4096,0.013667200009028116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,4096,0.0202346662680308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,4096,0.010609066486358643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,3584,0.012507733702659608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,3584,0.018956800301869713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,3584,0.01058240036169688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,3072,0.011338667074839274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,3072,0.009388800462086995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,3072,0.018633600076039633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,2560,0.008955732981363932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,2560,0.01761173407236735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,2560,0.008627200126647949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,2048,0.007733333110809326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,2048,0.016318933169047038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,2048,0.007335466643174489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,1536,0.006742399930953979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,1536,0.015655466914176942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,1024,0.004455466568470001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,1024,0.015421866377194723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,16384,0.027934932708740236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,768,0.0037546666959921518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,768,0.015135999520619711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,5120,0.012504532933235168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,512,0.0034474665919939675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,512,0.014863999684651694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,512,0.0051925331354141235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,256,0.0031797334551811217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,256,0.005085866649945577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,256,0.014707199732462563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,128,0.002869333326816559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,7168,0.015413332978884378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,6144,0.022420267264048256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,128,0.014428800344467163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,64,0.0027829334139823914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,32,0.0027989332874615988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,64,0.014521599809328715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,32,0.014335999886194864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,65536,0.07690133253733317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,65536,0.12074453035990398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,51200,0.09622080326080322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,51200,0.06166400114695231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,6144,0.014961066842079162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,16384,0.03331306576728821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,51200,0.07356906731923421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,16384,0.030653866132100423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,12288,0.026332799593607587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,12288,0.02680213252703349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,10240,0.022975999116897582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,10240,0.024935466051101685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,10240,0.019115734100341796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,8192,0.019156267245610557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,8192,0.022829866409301756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,1536,0.006466133395830791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,7168,0.017307732502619425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,7168,0.022111999988555908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,7168,0.014257066448529563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,6144,0.015313067038853965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,6144,0.021342933177947998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,768,0.0054517333706219995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,5120,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,5120,0.02017813324928284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,5120,0.012341333429018657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,4096,0.011686399579048157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,4096,0.018994132677714028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,4096,0.010760533809661865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,3584,0.01088213324546814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,3584,0.018500266472498576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,128,0.0047541335225105286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,3584,0.010345600048700969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,3072,0.010123733679453533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,3072,0.017350399494171144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,3072,0.00897813340028127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,2560,0.008310399949550629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,65536,0.09261866410573324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,2560,0.016200533509254454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,2048,0.0071370666225751235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,2048,0.016108799974123636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,1536,0.005532800157864889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,1536,0.01583573321501414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,1024,0.005899733304977417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,16384,0.027427200476328534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,1024,0.004156800111134847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,1024,0.015198933084805808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,12288,0.0218560000260671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,768,0.00383146678407987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,768,0.005457066496213277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,512,0.003455999990304311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,8192,0.016179200013478598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,512,0.015037866433461508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,512,0.005278933544953664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,256,0.003156266609827677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,256,0.0146506667137146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,256,0.005106133222579956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,128,0.0028138667345046996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,128,0.014455466469128927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,1024,0.005807999769846598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,64,0.0026943999032179515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,768,0.014884266257286071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,64,0.014632532993952433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,32,0.0027189334233601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,32,0.0144405335187912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,65536,0.09653546810150146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,65536,0.0629749337832133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,65536,0.08471466700236002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,6144,0.012943999965985617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,51200,0.07522772947947184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,16384,0.027446399132410686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,51200,0.05274666547775268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,51200,0.0667680025100708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,16384,0.027053866783777875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,16384,0.025140267610549927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,12288,0.021859200795491536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,12288,0.020218666394551596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,12288,0.02391466697057088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,10240,0.018984532356262206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,8192,0.015710933009783427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,10240,0.017794134219487508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,10240,0.022678399085998537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,7168,0.0143477330605189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,8192,0.015266133348147073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,8192,0.02116480072339376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,6144,0.012914133071899415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,7168,0.020835200945536293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,6144,0.012823466459910074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,6144,0.019947733481725058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,5120,0.011622400085131327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,5120,0.019590399662653604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,5120,0.011739733815193176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,4096,0.01014293332894643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,4096,0.009848533074061076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,3584,0.009478400150934856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,2560,0.008330666522185007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,2048,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,3584,0.016829866170883178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,3072,0.008906666437784832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,3072,0.016316800316174825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,2560,0.007642666498819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,1536,0.006621866424878438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,2560,0.016177067160606386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,2048,0.006459733347098033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,2560,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,2048,0.006920533378918965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,2048,0.015842133760452272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,1536,0.004752000172932943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,1536,0.0065749332308769224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,1536,0.015575466553370157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,1024,0.0040832000474135075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,1024,0.015197867155075073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,768,0.0037450666228930154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,768,0.014940800269444785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,768,0.005464533468087515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,512,0.0033301333586374915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,512,0.005100800096988678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,512,0.014745600024859109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,256,0.0030879999200503034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,256,0.014590932925542196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,256,0.0048650667071342465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,128,0.002844800055027008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,128,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,64,0.0027200000981489818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,64,0.014412800470987955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,32,0.0027669332921504974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,4096,0.018529067436854042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,32,0.01437226633230845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,128,0.004733866453170777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,65536,0.07442453702290854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,65536,0.052545066674550375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,51200,0.05948906739552816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,51200,0.04430720011393229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,16384,0.022055466969807945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,16384,0.024762666225433348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,16384,0.024829866488774617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,12288,0.01794559955596924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,12288,0.02211839954058329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,1024,0.005662933488686879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,12288,0.019937066237131755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,10240,0.01555519998073578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,10240,0.02091839909553528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,10240,0.01771519978841146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,8192,0.01323199967543284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,8192,0.014940800269444785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,8192,0.020387200514475505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,7168,0.012158933281898498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,7168,0.0200053334236145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,7168,0.013834666212399802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,6144,0.01129813293615977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,6144,0.012677333752314248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,6144,0.019001599152882895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,5120,0.010062932968139648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,5120,0.01879253387451172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,7168,0.014283733566602072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,4096,0.00904746651649475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,4096,0.017527467012405394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,51200,0.06632639964421591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,3584,0.008533333738644917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,3584,0.009618133306503296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,3584,0.009134933352470398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,3072,0.008060800035794576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,3072,0.00844373305638631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,3072,0.016456533471743265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,3072,0.008162133395671844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,2560,0.00720000018676122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,2560,0.01606613298257192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,2048,0.0056639999151229855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,2048,0.015698132912317912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,2048,0.0069567998250325514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,1536,0.004752000172932943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,1536,0.015422933300336204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,128,0.004681600133577982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,1024,0.004025600105524063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,65536,0.08410453001658122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,1024,0.015108266472816467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,1024,0.005816533168156942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,768,0.0037503999968369803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,768,0.015013333161671957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,512,0.0033962666988372804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,3584,0.016925867398579916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,512,0.014846932888031007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,512,0.005042133231957754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,256,0.0030005333324273427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,256,0.01460906664530436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,128,0.0027402666707833606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,128,0.014487466216087342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,128,0.004729599754015604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,64,0.002593066543340683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,64,0.014498133460680643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,32,0.002628266563018163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,32,0.014482133587201438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,65536,0.049661866823832196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,65536,0.040099199612935385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,65536,0.08329706986745199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,51200,0.03997439940770467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,51200,0.03495253324508667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,16384,0.015803733468055726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,51200,0.06720853646596273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,5120,0.011342933773994446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,16384,0.021653334299723305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,12288,0.013268267114957174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,12288,0.020080000162124634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,10240,0.011989333232243856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,10240,0.019495467344919838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,4096,0.009378133217493693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,8192,0.010457600156466167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,8192,0.018501333395640054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,2560,0.007573333382606506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,7168,0.009758933385213216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,7168,0.018078933159510292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,7168,0.013404800494511922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,6144,0.008703999718030294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,1536,0.006451199948787689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,6144,0.017280000448226928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,6144,0.01228053371111552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,5120,0.00830080012480418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,5120,0.010542933146158855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,5120,0.018051199118296304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,4096,0.007601066430409749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,4096,0.017219199736913045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,4096,0.00925439993540446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,3584,0.0070261334379514056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,3584,0.01675093372662862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,3584,0.008900266885757447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,3072,0.006411733229955037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,3072,0.016637866695721946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,2560,0.006028800209363302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,2560,0.016280532876650492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,768,0.005447466671466827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,2560,0.007607466479142506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,2048,0.005335466563701629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,2048,0.006854400038719177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,2048,0.015944533546765647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,1536,0.004747733473777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,1536,0.0156768004099528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,1536,0.006417066852251689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,1024,0.004016000032424927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,1024,0.015118933717409768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,256,0.004907733201980591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,768,0.003621333340803782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,768,0.005379199981689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,768,0.015000533064206442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,512,0.0034122665723164878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,512,0.005169066786766052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,512,0.014843733112017313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,256,0.003020799905061722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,256,0.014597333470980325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,128,0.0027647999425729113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,128,0.014469333489735923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,16384,0.025194666783014935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,12288,0.01989226738611857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,64,0.0026581334571043652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,64,0.0142794668674469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,10240,0.01731626590092977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,32,0.002696533252795537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,32,0.014409599701563516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,65536,0.039122132460276286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,65536,0.03513386646906535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,51200,0.03176106611887614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,51200,0.03063360055287679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,51200,0.06600533326466879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,16384,0.013219199577967324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,8192,0.01498240033785502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,16384,0.02103466590245565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,12288,0.01128426690896352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,12288,0.01880319913228353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,12288,0.019671465953191122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,10240,0.010361599922180175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,10240,0.01797013282775879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,10240,0.01707306702931722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,8192,0.00923520028591156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,8192,0.017145599921544394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,8192,0.014403200149536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,7168,0.008540800213813782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,3072,0.008074666559696197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,7168,0.017772799730300902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,6144,0.007786666850248973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,6144,0.01748159925142924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,6144,0.01167146662871043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,5120,0.007207466661930085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,5120,0.017885865767796834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,1024,0.005757866799831391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,5120,0.010466133554776508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,4096,0.006301866471767425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,4096,0.009274666508038838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,4096,0.01729493339856466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,3584,0.00650133341550827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,3584,0.016951467593510947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,3584,0.00895146628220876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,3072,0.006029866635799408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,3072,0.016577066977818807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,256,0.004871466755867004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,2560,0.006110933423042297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,2560,0.007978666822115581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,2560,0.016761600971221924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,2048,0.0053727999329566956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,2048,0.01623146633307139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,2048,0.006837333242098491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,1536,0.004700799783070883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,1536,0.015701333681742348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,128,0.004655999938646952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,1024,0.004040533304214477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,1024,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,768,0.0036277333895365395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,768,0.015038933356602988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,768,0.005342933535575867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,512,0.0033973333736260734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,512,0.014792533715566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,512,0.0051360001166661584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,256,0.0030080000559488933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,256,0.014603733023007711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,128,0.0028053333361943563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,128,0.014782933394114175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,128,0.004678399860858917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,64,0.002669866631428401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,64,0.014666666587193808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,65536,0.08386240005493165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,32,0.0026602665583292644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,65536,0.02765973409016927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,32,0.014176000157992044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,65536,0.02837226589520772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,51200,0.02290346622467041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,65536,0.08363306522369385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,51200,0.025301333268483477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,16384,0.010521599650382995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,16384,0.019364267587661743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,16384,0.024937599897384644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,16384,0.024626133839289348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,12288,0.0092031995455424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,12288,0.017259732882181803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,10240,0.008586666981379191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,10240,0.017930666605631508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,7168,0.013125333189964294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,10240,0.01693333387374878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,8192,0.007762133578459422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,8192,0.017578667402267455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,8192,0.01409386694431305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,7168,0.006559999783833821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,7168,0.017991467316945394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,6144,0.006157866617043813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,6144,0.017429333925247193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,6144,0.011590400338172912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,5120,0.00636053333679835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,5120,0.01807039976119995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,5120,0.010434133807818095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,3072,0.008040533463160197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,4096,0.005973333120346069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,4096,0.01725546717643738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,3584,0.0064064001043637585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,3584,0.016925867398579916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,51200,0.06626880168914795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,3584,0.008845866719881693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,3072,0.0061034664511680605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,3072,0.016471466422080992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,1536,0.0064640000462532045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,2560,0.006086400151252747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,12288,0.01955733299255371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,2560,0.007701333363850911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,1024,0.005787733197212219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,2560,0.016102400422096253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,2048,0.0053269331653912864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,1536,0.004636799792448679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,2048,0.015800533692042033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,1536,0.006477866570154827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,1536,0.01578986644744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,1024,0.004056533426046371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,1024,0.005658666789531708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,1024,0.01524906655152639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,768,0.003719466676314672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,768,0.014972800016403198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,768,0.0054730668663978575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,512,0.003409066547950109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,512,0.014762666821479798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,512,0.0050005331635475155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,256,0.003094399968783061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,256,0.014612266421318054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,256,0.004845866560935974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,128,0.0027637332677841187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,256,0.00487253318230311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,64,0.002721066772937775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,128,0.014482133587201438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,32,0.002679466704527537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,64,0.014342400431632995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,32,0.014297599593798319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,65536,0.017206400632858276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,65536,0.02321066657702128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,51200,0.01461013356844584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,51200,0.021577600638071695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,51200,0.06619306802749633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,16384,0.0077344000339508055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,16384,0.01762666702270508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,2048,0.0067893331249554946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,12288,0.006316799918810527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,12288,0.018345600366592406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,7168,0.012964266538619994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,10240,0.006296533346176148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,10240,0.01773973306020101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,10240,0.016432000199953715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,8192,0.006074666480223338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,8192,0.017126399278640746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,8192,0.01406719982624054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,7168,0.0060703997810681665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,4096,0.009212799866994222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,7168,0.01794346570968628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,7168,0.012843733032544455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,6144,0.0058783998092015585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,6144,0.011702400445938111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,6144,0.01750613252321879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,5120,0.0063391998410224915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,5120,0.018065067132314046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,4096,0.005857066810131073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,4096,0.017566933234532674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,4096,0.009677867094675701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,3072,0.008017066617806752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,3584,0.01698346734046936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,3072,0.006018133461475372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,3072,0.016643200318018594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,3072,0.007978666822115581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,2560,0.006065066655476888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,2560,0.016251732905705772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,2560,0.007619200150171916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,2048,0.005352533360322317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,2048,0.0160181333621343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,2048,0.0067104001839955645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,1536,0.004658133288224538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,1536,0.015558399756749473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,1536,0.0063701331615448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,1024,0.004030933231115341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,1024,0.015016532937685647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,128,0.004637866715590159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,768,0.0036917333801587426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,65536,0.08331200281778971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,768,0.014974932869275412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,512,0.0032885332902272543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,512,0.014814933141072592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,512,0.004951466619968414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,256,0.0029663999875386557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,3584,0.006320000191529592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,256,0.014477866888046264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,256,0.0048767998814582825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,128,0.002746666719516118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,128,0.014313600460688271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,128,0.004712533454100291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,64,0.002677333354949951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,16384,0.02380160093307495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,32,0.0026975999275843303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,64,0.014265599846839904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,65536,0.011725866794586181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,32,0.014365866780281067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,65536,0.020004266500473024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,51200,0.010317867000897724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,51200,0.018550399939219156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,12288,0.018939733505249023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,16384,0.00589333325624466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,51200,0.06559146642684936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,16384,0.018568533658981323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,16384,0.023746132850646973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,12288,0.005924266576766968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,12288,0.017280000448226928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,10240,0.006142933170000712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,10240,0.017570134003957114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,5120,0.010435199737548828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,8192,0.005973333120346069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,8192,0.017142399152119955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,3584,0.008835200468699138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,7168,0.005884799857934316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,7168,0.017987199624379478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,7168,0.012810666362444559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,6144,0.00574186642964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,6144,0.017318399747212727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,1024,0.005691733459631602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,5120,0.0059562668204307554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,5120,0.01763733426729838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,768,0.0053375999132792154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,4096,0.0056202664971351625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,4096,0.017010132471720375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,4096,0.00920746624469757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,3584,0.006061866879463196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,3584,0.01687893271446228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,3584,0.008713600039482117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,3072,0.005758933226267497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,3072,0.016473600268363954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,3072,0.007963733375072479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,65536,0.08273599942525228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,2560,0.005731200178464254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,2560,0.015964800119400026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,2560,0.007619200150171916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,2048,0.0050346667567888895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,2048,0.006743466854095459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,2048,0.015749333302179973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,1536,0.004533333579699198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,12288,0.018917334079742432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,1536,0.015352533260981242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,1536,0.0063178668419520065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,1024,0.003832533210515976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,768,0.00360959991812706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,1024,0.005568000177542368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,1024,0.015239466230074564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,512,0.0032831999162832894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,768,0.005350400010744731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,10240,0.01644373337427775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,768,0.015020799636840821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,256,0.0029237332443396253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,512,0.005031466484069824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,128,0.0027701333165168762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,256,0.00480320006608963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,256,0.014365866780281067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,128,0.014313600460688271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,64,0.0026911998788515727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,32,0.0026591998835404714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,64,0.014375467101732889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,65536,0.00844586690266927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,32,0.01439466675122579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,51200,0.007788800199826558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,65536,0.018467199802398682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,5120,0.01032960017522176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,16384,0.01772586703300476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,12288,0.005721599857012431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,10240,0.005869866907596588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,12288,0.016978132724761962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,8192,0.005875200033187866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,10240,0.017505067586898803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,8192,0.017402666807174682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,7168,0.005658666789531708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,6144,0.005576533575852713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,7168,0.017468800147374473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,5120,0.00602346658706665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,6144,0.017102932929992674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,4096,0.005558399856090546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,5120,0.017560533682505288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,4096,0.016829866170883178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,3584,0.006083199878533682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,8192,0.01399893363316854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,3072,0.0057429333527882894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,3584,0.016503467162450155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,2560,0.005707733333110809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,3072,0.016681599617004394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,2048,0.01574186682701111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,1536,0.004423466821511587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,1536,0.01535040040810903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,1024,0.003765333443880081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,1024,0.01482133368651072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,512,0.014886400103569031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,768,0.0034634667138258614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,6144,0.01165226697921753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,512,0.0032479998966058097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,768,0.014948266744613647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,256,0.0028970666229724885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,512,0.01436906655629476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,128,0.0026922665536403658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,256,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,128,0.014074666301409402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,16384,0.0057546665271123254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,64,0.002632533262173335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,51200,0.01888426740964254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,32,0.002584533393383026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,32,0.014299733440081277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,65536,0.007816533247629803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,51200,0.007665066421031952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,65536,0.019307732582092285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,51200,0.018863999843597413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,16384,0.005701333284378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,12288,0.005799466868241628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,16384,0.01752106746037801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,10240,0.005861333509286245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,12288,0.017203199863433837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,8192,0.005795200169086456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,10240,0.01794346570968628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,7168,0.005678933362166087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,8192,0.017157334089279174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,7168,0.01752106746037801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,6144,0.005573333303133646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,5120,0.0058783998092015585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,6144,0.017303466796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,4096,0.005475200215975444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,5120,0.017778132359186807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,3584,0.0059562668204307554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,4096,0.01703146696090698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,3072,0.005610666672388713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,3584,0.016921599706014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,2560,0.005681066711743673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,3072,0.01627413332462311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,2048,0.0050570666790008545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,2048,0.005029333134492239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,1536,0.004420266548792521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,2048,0.016059733430544534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,1024,0.0037439999481042228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,1536,0.015326933066050211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,768,0.0034624000390370687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,1024,0.01513920029004415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,512,0.0031839999059836066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,768,0.014769066373507181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,256,0.0029258665939172106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,512,0.014673067132631936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,256,0.014421332875887552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,64,0.0026122666895389557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,128,0.014421332875887552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,32,0.002616533388694127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,64,0.01432960033416748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,32,0.014339199662208557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,64,0.014197333653767904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,128,0.0046858668327331545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,2560,0.01598186691602071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,128,0.0027413333455721537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,16384,0.7469621022542318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,16384,0.39193172454833985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,16384,0.35508267084757483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,12288,0.5672938664754231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,2560,0.015848533312479655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,12288,0.2970730781555176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,51200,1.1797845204671225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,51200,1.0943871815999349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,10240,0.2547370592753092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,10240,0.2243893305460612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,10240,0.4726858774820964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,51200,2.338281504313151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,8192,0.2217930634816488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,7168,0.16842026710510255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,8192,0.2111776034037272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,7168,0.17997013727823893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,7168,0.3295381228129069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,8192,0.3792682647705078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,6144,0.15655253728230795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,6144,0.13726506233215333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,5120,0.23846294085184733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,5120,0.12214079697926838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,4096,0.19242026011149088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,6144,0.3362037340799967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,3584,0.09834986527760824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,4096,0.120632537206014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,4096,0.09372586409250895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,3584,0.08853440284729004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,3584,0.17561920483907062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,3072,0.0866218646367391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,3072,0.16964799563090008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,2560,0.120906662940979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,3072,0.07189013163248698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,2560,0.061305598417917884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,2560,0.07475519975026448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,2048,0.09691627025604248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,1536,0.07464213371276855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,1536,0.03953386545181274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,1536,0.05153066714604696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,1024,0.05903893311818441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,2048,0.07103359699249268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,1024,0.03941973447799683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,768,0.039401598771413165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,512,0.02728213270505269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,768,0.033446399370829265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,768,0.025125332673390705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,512,0.027000532547632856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,512,0.01703146696090698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,256,0.012179199854532878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,256,0.02077546715736389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,128,0.008980266253153483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,128,0.019631999731063842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,64,0.0062282666563987735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,64,0.0184661328792572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,32,0.00450133333603541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,32,0.018388267358144125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,65536,5120,0.13311999638875324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,2048,0.05071253379185995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,12288,0.26799147923787436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,65536,256,0.015352533260981242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,65536,1.2139755249023438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,1024,0.028775467475255327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,51200,0.9560266494750976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,65536,128,0.009537067015965779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,65536,1.1541812896728516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,65536,2.362523651123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,51200,1.8530378977457684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,16384,0.3513408025105794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,16384,0.6041482925415039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,16384,0.29499092102050783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,12288,0.5111167907714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,10240,0.18616639773050944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,12288,0.27424532572428384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,10240,0.37683626810709636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,10240,0.1992970625559489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,12288,0.25809812545776367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,8192,0.30805120468139646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,8192,0.16394774119059247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,7168,0.13243093490600585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,7168,0.16543893814086913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,7168,0.2656405289967855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,6144,0.1354570706685384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,6144,0.22729600270589195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,5120,0.10846186478932698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,5120,0.09627733230590821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,5120,0.18688534100850424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,4096,0.15321173667907714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,4096,0.08908373514811198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,4096,0.07814719676971435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,3584,0.06970453262329102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,3584,0.13451840082804362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,3584,0.07947306632995606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,3072,0.1140554666519165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,3072,0.07093653678894044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,3072,0.06045973300933838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,2560,0.05160106817881266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,2560,0.09698773225148519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,2560,0.062108798821767175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,2048,0.07721813519795737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,1536,0.05772480169932047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,1536,0.04262719949086507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,1024,0.04108800093332927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,1024,0.033650132020314534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,1024,0.024734934171040855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,768,0.031137067079544067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,51200,0.902732785542806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,768,0.028497066100438433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,768,0.01983039975166321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,512,0.023667200406392416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,512,0.02440746625264486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,256,0.012251733740170797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,256,0.011057066917419433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,256,0.01998400092124939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,128,0.007632000247637431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,128,0.007718400160471599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,128,0.015117866794268289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,64,0.003995733211437861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,51200,32,0.004040533304214477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,64,0.015074132879575094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,32,0.015542399883270264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,51200,2048,0.05260693232218424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,2048,0.04784853458404541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,65536,0.381984011332194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,65536,0.7400138854980469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,51200,0.30297600428263344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,65536,0.3549621264139811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,8192,0.15041066805521647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,51200,0.5809194564819335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,16384,0.18619200388590496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,16384,0.10762773354848225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,16384,0.09333653450012207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,12288,0.16618879636128742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,12288,0.07199040253957113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,12288,0.09741013050079346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,6144,0.11479679743448894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,10240,0.06074026823043823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,10240,0.07290773391723633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,8192,0.061592535177866614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,8192,0.049670398235321045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,7168,0.08282559712727865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,7168,0.055934933821360264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,7168,0.0443125327428182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,6144,0.07235626379648843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,6144,0.0503114660580953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,5120,0.06018346548080444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,5120,0.044514131546020505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,5120,0.03742826779683431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,1536,0.03760639826456706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,4096,0.04914773305257161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,4096,0.03844906489054362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,4096,0.027714133262634277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,51200,512,0.015241600076357522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,3584,0.035854931672414145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,3584,0.025290666023890178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,3072,0.037647998332977294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,3072,0.0327455997467041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,2560,0.031941332419713336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,2560,0.029943466186523438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,2560,0.019898666938145956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,10240,0.11657813390096028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,2048,0.02606719930966695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,2048,0.028867199023564653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,8192,0.09490666389465333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,1536,0.020629332462946574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,1536,0.023476266860961915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,1536,0.013608533143997192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,1024,0.01458453337351481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,1024,0.02070186734199524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,768,0.011513599753379821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,768,0.019363200664520262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,3584,0.04332480033238729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,512,0.00864533285299937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,512,0.01780160069465637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,512,0.007624533275763195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,256,0.005820799867312113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,256,0.015277866522471109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,256,0.005547733108202616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,128,0.003538133452335993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,128,0.014934399724006652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,51200,0.2875125249226888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,64,0.0031680000325044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,64,0.014968533317248026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,16384,32,0.003302400062481562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,16384,32,0.014843733112017313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,6144,0.03906986713409424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,65536,0.5542922973632812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,65536,0.29348694483439125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,3072,0.02245866656303406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,2048,0.016477866967519125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,65536,0.2861088116963705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,51200,0.23439040184020996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,51200,0.434665584564209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,16384,0.144268798828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,16384,0.08740479946136474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,12288,0.06761279900868734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,12288,0.05796480178833008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,12288,0.1291050672531128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,10240,0.058533334732055665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,10240,0.09236053625742593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,768,0.009250133236249288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,10240,0.04834666649500529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,8192,0.05052479902903238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,8192,0.0757269303003947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,8192,0.0403498649597168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,7168,0.06657706499099732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,7168,0.045774932702382407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,6144,0.054034133752187095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,7168,0.03620586792627971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,6144,0.041307731469472246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,5120,0.04532586733500163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,6144,0.03204586704572042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,5120,0.036728533109029134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,4096,0.03688853184382121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,4096,0.03221120039621989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,3584,0.032579199473063154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,3584,0.030485333998998006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,3584,0.021196800470352172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,3072,0.02887679934501648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,3072,0.027937066555023194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,2560,0.024843732515970864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,2560,0.025510400533676147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,2560,0.016795732577641807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,2048,0.020361600319544475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,128,0.005111466844876607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,2048,0.02355413238207499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,2048,0.01376106639703115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,1536,0.016942934195200602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,1536,0.021604265769322714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,1536,0.011718400319417318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,1024,0.011640533804893494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,1024,0.019339734315872194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,1024,0.009272533655166625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,768,0.009831466277440389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,16384,1024,0.010709333419799804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,512,0.007493333518505096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,768,0.008136533200740814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,512,0.005658666789531708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,512,0.015159466862678527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,256,0.004063999901215235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,256,0.0151637335618337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,128,0.0034602666894594826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,128,0.014857600132624308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,768,0.01812160015106201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,64,0.003270400067170461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,64,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,12288,32,0.0032416000962257386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,51200,0.2205354690551758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,16384,0.07558293342590332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,12288,32,0.015223466356595359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,5120,0.027332266171773274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,4096,0.02313493291536967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,65536,0.2528245290120443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,65536,0.4766890525817871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,3072,0.018764799833297728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,16384,0.12427626450856526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,51200,0.20161706606547036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,51200,0.3746335983276367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,16384,0.0765941301981608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,16384,0.06402026812235515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,12288,0.05943893194198609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,12288,0.04905279874801636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,10240,0.07891093095143636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,10240,0.051826135317484534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,8192,0.0647648016611735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,256,0.005274666845798493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,8192,0.0442197322845459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,8192,0.03481493393580119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,7168,0.053564798831939694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,7168,0.04074560006459554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,7168,0.03094293276468913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,6144,0.047023999691009524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,6144,0.03712533315022786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,6144,0.02750399907430013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,5120,0.039290666580200195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,5120,0.033530668417612715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,5120,0.023988266785939537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,4096,0.03192960023880005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,4096,0.029495465755462646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,4096,0.020528000593185425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,3584,0.02849493424097697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,3584,0.02821333408355713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,3584,0.018617600202560425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,3072,0.024819199244181314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,3072,0.025684267282485962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,2560,0.021172267198562623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,2560,0.023705599705378215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,12288,0.09849812984466552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,2048,0.01750719944636027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,2048,0.02198293407758077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,2048,0.012637866536776224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,1536,0.013860266407330832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,1536,0.010808533430099488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,1536,0.02050666610399882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,1024,0.010054399569829304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,1024,0.008724266290664673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,1024,0.01848319967587789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,768,0.008365866541862488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,768,0.007155199845631917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,512,0.006679466863473256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,512,0.01504853367805481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,65536,0.23713173866271972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,51200,0.18641707102457683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,256,0.003550933301448822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,256,0.005118933320045471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,256,0.01421333352724711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,128,0.003235200047492981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,128,0.014672000209490457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,128,0.00494400014479955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,64,0.0029290666182835894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,10240,32,0.002922666569550832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,12288,128,0.004996266464392344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,64,0.014646400014559427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,32,0.01474666694800059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,10240,768,0.015833600362141927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,65536,0.20097813606262208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,65536,0.18473493258158366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,65536,0.37920427322387695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,51200,0.2952095985412598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,16384,0.10228906472524006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,51200,0.1455626646677653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,51200,0.16147947311401367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,16384,0.06847146352132162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,16384,0.05063786506652832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,12288,0.0704085350036621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,3072,0.016565333803494772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,12288,0.03997439940770467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,12288,0.0495743989944458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,10240,0.05911786556243896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,8192,0.04791040023167928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,10240,0.03395946820576985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,8192,0.03932160139083862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,8192,0.02844799955685933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,7168,0.04225600163141886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,7168,0.03515413204828898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,7168,0.025665066639582318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,6144,0.03738773266474406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,6144,0.03242879907290141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,5120,0.03143466711044311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,5120,0.02943146626154582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,6144,0.022817067305246987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,10240,0.04179840087890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,4096,0.025722666581471758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,4096,0.026234666506449383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,3584,0.02294399936993917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,3584,0.015448533495267234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,3584,0.02480319937070211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,3072,0.020214400688807168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,3072,0.02336533268292745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,3072,0.013736533125241599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,2560,0.017382399241129557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,2560,0.02222613294919332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,2560,0.012444800138473511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,2048,0.01432319978872935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,2048,0.011004799604415893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,2048,0.020498132705688475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,1536,0.01157866617043813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,1536,0.019197867314020792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,1024,0.008574933807055155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,1024,0.017347200711568197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,512,0.005666133264700572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,1024,0.007597866654396057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,768,0.007245866457621257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,768,0.015340800086657206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,768,0.006190933287143707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,512,0.005676800012588501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,512,0.01479573349157969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,512,0.005323733389377594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,256,0.0034656000634034475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,256,0.014747732877731323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,256,0.005212800204753875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,128,0.0031445334355036415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,10240,0.04369599819183349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,64,0.0029311999678611755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,128,0.014578133821487427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,8192,32,0.002940800040960312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,64,0.01455573340257009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,10240,2560,0.014660267035166421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,8192,32,0.014613333344459533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,65536,0.1630741278330485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,65536,0.18340373039245605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,65536,0.3267296155293783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,51200,0.14837973912556965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,51200,0.12959252993265788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,51200,0.25820906956990564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,16384,0.0567840019861857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,16384,0.09008320172627768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,16384,0.045686399936676024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,12288,0.06840960184733072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,12288,0.047126400470733645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,12288,0.03596160014470418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,10240,0.05482986768086752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,10240,0.041492267449696855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,10240,0.030714666843414305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,8192,0.03547093470891317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,7168,0.0400000015894572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,7168,0.03320639928181966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,7168,0.02322773337364197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,6144,0.03541333278020223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,6144,0.03091199994087219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,4096,0.01716586748758952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,5120,0.019896533091862997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,6144,0.02071573336919149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,5120,0.030035199721654256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,5120,0.018041600783665977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,5120,0.028137600421905516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,4096,0.02318293253580729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,4096,0.01562773287296295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,4096,0.025271467367808026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,3584,0.020746666193008422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,3584,0.014174933234850565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,3584,0.023788799842198692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,3072,0.018380800882975258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,2560,0.01583466629187266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,3072,0.022653865814208984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,2560,0.011796266833941142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,2560,0.021418666839599608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,2048,0.01318933367729187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,1536,0.009773866335550944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,2048,0.010211199522018433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,2048,0.019885865847269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,1536,0.010744532942771912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,1536,0.009257599711418152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,1536,0.01883626580238342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,1024,0.00805866668621699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,1024,0.0069567998250325514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,1024,0.015397333105405173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,8192,0.04591999848683675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,768,0.01511360009511312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,512,0.004043733328580856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,512,0.01518933375676473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,256,0.003304533412059148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,256,0.00520000010728836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,256,0.014819199840227762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,128,0.003173333406448364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,128,0.014662399888038635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,128,0.004932266473770141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,64,0.002865066627661387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,32,0.0029333333174387617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,64,0.01470186710357666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,7168,32,0.014454399545987448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,65536,0.2806495984395345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,65536,0.15667732556660968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,3072,0.012874666849772134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,51200,0.22168639500935874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,51200,0.12584959665934245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,7168,768,0.006858666737874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,16384,0.07524693012237549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,51200,0.11372906366984051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,16384,0.05163520177205404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,12288,0.057608532905578616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,8192,0.02574613293011983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,8192,128,0.004783999919891357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,10240,0.049394134680430093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,10240,0.03743786811828613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,10240,0.02753173311551412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,8192,0.03902186552683513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,8192,0.033521068096160886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,8192,0.023001599311828613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,7168,0.03409706751505534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,7168,0.031220267216364544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,12288,0.04222079912821452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,768,0.006126933296521505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,7168,512,0.005564799904823304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,6144,0.029013333717981975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,5120,0.025577600797017413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,5120,0.02725333372751872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,65536,0.1473632017771403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,4096,0.02416426738103231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,4096,0.02452053427696228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,4096,0.014197333653767904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,6144,0.029978666702906293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,3584,0.019138133525848387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,6144,0.018902399142583213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,3584,0.01297813355922699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,3584,0.022989867130915324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,3072,0.016784000396728515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,3072,0.011789866288503011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,3072,0.022141865889231362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,12288,0.03219839930534363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,2560,0.020603734254837035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,16384,0.040164268016815184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,2048,0.019332265853881835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,1536,0.011069867014884948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,1536,0.01812053322792053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,1536,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,1024,0.007383466760317485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,1024,0.006362666686375936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,1024,0.015024000406265258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,768,0.00633493314186732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,768,0.015384533007939658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,768,0.005869866907596588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,512,0.003769599894682566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,512,0.014882133404413859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,256,0.0032085334261258446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,256,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,256,0.005102933446566264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,128,0.003070933371782303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,128,0.014484266440073649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,128,0.004795733094215393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,64,0.0027935999135176343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,7168,0.020973867177963255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,64,0.014484266440073649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,32,0.0028042666614055633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,6144,32,0.01446399986743927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,2560,0.014691199858983359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,65536,0.1371658643086751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,65536,0.23900160789489747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,6144,2048,0.012360533078511555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,65536,0.12408426602681477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,51200,0.0988981326421102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,51200,0.10781546433766682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,51200,0.18889387448628742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,16384,0.04546986818313599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,16384,0.06332053343454996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,12288,0.048706134160359696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,5120,0.016463999946912132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,12288,0.03720959822336833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,12288,0.02858453392982483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,10240,0.041867733001708984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,8192,0.033780264854431155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,10240,0.03360746701558431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,8192,0.02988479932149251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,8192,0.020362667242685952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,7168,0.028060799837112425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,7168,0.028076799710591634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,7168,0.018786134322484334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,6144,0.02482879956563314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,6144,0.01679253379503886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,6144,0.026125866174697875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,5120,0.02127359906832377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,5120,0.014706133802731832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,5120,0.02408213416735331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,4096,0.01269760032494863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,4096,0.022601600488026938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,3584,0.015787733594576518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,3584,0.021527467171351115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,3584,0.012040533622105916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,3072,0.014032000303268432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,3072,0.020552533864974975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,2560,0.012123733758926392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,2560,0.019528534015019736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,2560,0.010294399658838908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,2560,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,2048,0.010401067137718201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,2048,0.018776534001032512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,1536,0.008700799942016602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,2048,0.00951039989789327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,16384,0.03633600076039632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,1536,0.016937599579493205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,1024,0.006748799979686737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,1536,0.007190399865309398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,1024,0.005801600217819214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,1024,0.015255467096964518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,768,0.005251200000445048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,768,0.015340800086657206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,512,0.003606399893760681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,512,0.015033599734306336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,512,0.0053610667586326596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,256,0.0031498665610949195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,4096,0.017565866311391197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,256,0.014919466773668923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,6144,512,0.005351466437180838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,128,0.002974933385848999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,128,0.01448319951693217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,64,0.002809600035349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,64,0.014507733782132468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,5120,32,0.0028181334336598715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,5120,32,0.014733866850535075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,65536,0.1868010679880778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,65536,0.10950613021850586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,65536,0.10644906361897785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,51200,0.1479082743326823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,10240,0.024647466341654458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,256,0.004996266464392344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,16384,0.04991039832433065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,51200,0.0889141321182251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,51200,0.08526933193206787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,16384,0.039211734135945635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,12288,0.0390559991200765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,12288,0.033404799302419026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,10240,0.03312746683756511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,3072,0.010860799749692281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,10240,0.030753066142400105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,10240,0.021735467513402305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,8192,0.027534933884938557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,8192,0.018284799655278523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,8192,0.027475200096766156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,7168,0.024729599555333458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,7168,0.02540266712506612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,7168,0.016614400347073875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,6144,0.022087466716766358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,2048,0.008962133526802063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,6144,0.02373759945233663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,5120,0.019115734100341796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,5120,0.013357866803805032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,5120,0.02256106734275818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,4096,0.01590506633122762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,4096,0.020914133389790854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,3584,0.013286399841308593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,3584,0.02015786568323771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,3584,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,3072,0.011771733562151592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,3072,0.019667200247446694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,2560,0.010291199882825215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,2560,0.019001599152882895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,768,0.005523199836413065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,2560,0.00960106650988261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,2048,0.008777599533398945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,2048,0.008247466882069905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,2048,0.017886932690938315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,1536,0.007580799857775371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,1536,0.015615999698638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,1536,0.006625066697597504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,1024,0.0059690664211908976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,1024,0.015196800231933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,1024,0.005960533519585928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,5120,128,0.004861866434415182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,768,0.0037759999434153237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,512,0.0034613333642482757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,768,0.005614933371543884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,768,0.015264000495274863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,256,0.0031829332311948144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,512,0.005245866874853769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,512,0.014842666188875833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,256,0.004886400202910105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,128,0.0029685333371162414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,128,0.004666666686534882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,128,0.01470080018043518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,64,0.0027200000981489818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,4096,32,0.0028042666614055633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,64,0.01467519998550415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,6144,0.014852266510327658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,32,0.014366933703422546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,65536,0.16847039858500162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,65536,0.10079893271128337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,51200,0.08324266274770101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,4096,256,0.01492586632569631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,12288,0.02490880091985067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,16384,0.031229867537816362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,16384,0.04774186611175537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,16384,0.03644479910532634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,12288,0.03499093453089396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,12288,0.031090132395426434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,10240,0.029926399389902752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,10240,0.028777599334716797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,10240,0.02106346686681112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,51200,0.13098666667938233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,4096,0.011708799997965496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,8192,0.024716800451278685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,8192,0.025406932830810545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,7168,0.022478934129079184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,7168,0.024280534187952677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,4096,3072,0.010196266571680705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,6144,0.019822933276494346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,6144,0.02317333420117696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,6144,0.014541866381963095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,5120,0.017428267002105712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,5120,0.021319466829299926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,5120,0.012970667084058127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,4096,0.013265066345532737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,4096,0.020118399461110433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,65536,0.09762453238169352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,3584,0.013190399607022604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,3584,0.019513599077860513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,3584,0.01060693363348643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,3072,0.01088853379090627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,51200,0.08181227048238118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,3072,0.019221333662668864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,3072,0.010020266969998677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,2560,0.00962666670481364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,2560,0.018838399648666383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,2048,0.008335999647776286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,2048,0.016204800208409628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,16384,0.02983466585477193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,1536,0.007177599767843883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,12288,0.024152533213297526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,1536,0.006857599814732869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,1536,0.01564906636873881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,1024,0.004796800017356872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,1024,0.015321600437164306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,768,0.003833599885304769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,768,0.01515733301639557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,768,0.005674666663010915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,512,0.003502933432658514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,512,0.005223466455936432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,512,0.014897066354751586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,256,0.003197866678237915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,7168,0.01621333360671997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,256,0.014667733510335287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,256,0.005117866893609365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,128,0.0029653333127498626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,128,0.004779733220736186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,64,0.0028021333118279776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,128,0.014314666390419006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3584,32,0.0027232001225153605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,64,0.014547200004259745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3584,32,0.014350933829943338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,65536,0.14383680025736492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,65536,0.0879957358042399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,65536,0.094540802637736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,51200,0.07209493319193522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,51200,0.11378560066223145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,51200,0.07525653044382731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,16384,0.040150400002797446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,12288,0.03151893417040507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,16384,0.03368746836980184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,4096,0.011545600493748982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,12288,0.028919466336568195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,10240,0.026710400978724163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,10240,0.026909865935643512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,8192,0.022431999444961548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,8192,0.0248799999554952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,8192,0.016500266393025716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,7168,0.019949867328008016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,8192,0.017590399583180746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,7168,0.02369706630706787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,7168,0.01528320014476776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,6144,0.01756160060564677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,6144,0.02241493264834086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,6144,0.013873066504796347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,5120,0.015442132949829102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,5120,0.020858667294184365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,5120,0.01243839959303538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,4096,0.013822933038075766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,4096,0.019798400004704793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,4096,0.010985599954922994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,3584,0.012593066692352295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,3584,0.019100799163182577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,3072,0.011400533715883891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,2560,0.00914026697476705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,3072,0.018524799744288126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,3072,0.0093941330909729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,2560,0.008959999680519104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,2048,0.008106666803359985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,2048,0.007736533383528392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,2048,0.015979733069737753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3584,1024,0.00613013356924057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,1536,0.006822399795055389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,1536,0.01576959987481435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,1536,0.006595199803511302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,1024,0.0041514667371908825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,1024,0.015416533748308817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,1024,0.005871999760468801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,768,0.0038730666041374207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,768,0.015251200397809347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,768,0.005646933118502299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,2560,0.017717333634694417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,512,0.003656533360481262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,512,0.014824533462524414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,12288,0.02256960074106852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,16384,0.02805759906768799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,256,0.003197866678237915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,256,0.004947199920813243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,10240,0.019550933440526327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,128,0.002887466549873352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,128,0.004886400202910105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,64,0.0027690666417280836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,64,0.014553599556287131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,3072,32,0.002780800064404805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,32,0.014600533246994018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,65536,0.12034666538238525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,65536,0.07667733033498128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,3584,0.010638933380444844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,51200,0.09588479995727539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,2560,0.00886826713879903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,51200,0.06180373430252075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,2048,0.0075647999842961625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,51200,0.07375360329945882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,16384,0.030598400036493938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,16384,0.027585067351659137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,12288,0.026293333371480303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,12288,0.0265610675017039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,12288,0.02207039992014567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,10240,0.022809600830078124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,10240,0.024706133206685386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,8192,0.01913493275642395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,8192,0.022785067558288574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,65536,0.09273173014322916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,7168,0.01730133295059204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,7168,0.022111999988555908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,16384,0.03336639801661174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,7168,0.015202132860819497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,3072,512,0.005224533379077911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,6144,0.013552000125249227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,6144,0.021208532651265464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,5120,0.013524267077445983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,5120,0.02049386699994405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,4096,0.011725866794586181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,256,0.01479039986928304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,4096,0.01040000021457672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,4096,0.01909653345743815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,3584,0.010925867160161336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,3584,0.01035093367099762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,3584,0.01867199937502543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,3072,0.010136533776919048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,3072,0.00897706647713979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,3072,0.01747093399365743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,2560,0.008289066453774769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,2048,0.007179733117421467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,2560,0.016371200482050575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,2048,0.0068906664848327635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,2048,0.015828266739845276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,1536,0.005643733342488607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,1536,0.006490666667620342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,1536,0.015726932883262636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,1024,0.00415040006240209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,1024,0.005771733323733012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,1024,0.015336533387502035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,768,0.003705599904060364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,768,0.015235199530919393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,512,0.0034453332424163817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,512,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,512,0.005169066786766052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,256,0.0031413334111372627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,256,0.01455466647942861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,256,0.005074133475621542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,128,0.0028970666229724885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,6144,0.015482667088508605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,128,0.014519466956456503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,64,0.002737066646416982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,64,0.014429866274197897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2560,32,0.0027647999425729113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,3072,128,0.014545067151387533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2560,32,0.014407466848691305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,2560,0.008323200047016144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,65536,0.062310401598612467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,65536,0.09653440316518148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,51200,0.0525322675704956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,51200,0.07513386408487956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,65536,0.08559146722157797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,10240,0.01906986633936564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,16384,0.02749226689338684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,16384,0.02767360011736552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,16384,0.02540053327878316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,12288,0.02189226746559143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,8192,0.01636799971262614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,51200,0.06794026692708334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,12288,0.020371200640996297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,10240,0.019064533710479736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,8192,0.015684266885121666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,10240,0.02286826570828756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,8192,0.021350399653116862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,8192,0.015412267049153647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,7168,0.014235732952753702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,7168,0.02095680038134257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,6144,0.012806399663289388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,6144,0.01991999944051107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,6144,0.013005866607030233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,5120,0.011770666639010111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,5120,0.01956693331400553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,5120,0.011709866921106975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,4096,0.010194133718808491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,4096,0.018713599443435668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,5120,0.012316800157229106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,3584,0.0095360000928243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,3584,0.017061332861582436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,3584,0.009532800316810608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,3072,0.008865066369374593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,3072,0.01644266645113627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,12288,0.02405866583188375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,3072,0.00853760043780009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,2560,0.0076447998483975725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,2560,0.016268799702326454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,7168,0.014390400052070618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,2048,0.006400000055631001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,2048,0.015871999661127727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,2048,0.006988800068696339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,1536,0.004700799783070883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,1536,0.015501866738001505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,768,0.005465599894523621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,1024,0.003992533435424169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,1024,0.015146666765213012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,768,0.003702399879693985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,768,0.015013333161671957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2560,128,0.004785066843032837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,512,0.0033333333830038703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,512,0.014969600240389505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,512,0.005131733417510986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,256,0.0031178665657838186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,256,0.014607999722162882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,256,0.004947199920813243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,128,0.0028714666763941447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,2560,0.007915733257929485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,128,0.01462399959564209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,64,0.0027093333502610523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,1536,0.006551466882228851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,64,0.014443733294804893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,2048,32,0.002757333219051361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,10240,0.017802667617797852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,2048,32,0.014406399925549826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,65536,0.07430720329284668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,65536,0.0521237333615621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,51200,0.05941333373387655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,51200,0.044343467553456625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,16384,0.022106667359670006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,51200,0.06675733725229899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,16384,0.024330667654673257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,12288,0.018016000588734947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,12288,0.02260266741116842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,10240,0.015453867117563882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,10240,0.0214303990205129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,10240,0.01748266617457072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,8192,0.013186132907867432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,8192,0.02026453415552775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,4096,0.010275200009346008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,7168,0.012238933642705282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,7168,0.01967573364575704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,7168,0.0139957328637441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,6144,0.011238400141398113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,128,0.004646400113900503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,6144,0.019160532951354982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,6144,0.012804266810417176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,5120,0.010201600193977357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,5120,0.019338667392730713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,4096,0.009066667159398396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,4096,0.017423999309539796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,4096,0.009439999858538311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,3584,0.008558932940165203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,1024,0.005712000032265982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,3584,0.01699733336766561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,3072,0.008067200084527333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,3072,0.016668800512949625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,3072,0.008072533210118612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,2560,0.007017600039641063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,2048,768,0.005381333331267038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,2560,0.007892266909281413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,2560,0.016246400276819863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,2048,0.005846400062243144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,2048,0.016039466857910155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,2048,0.006799999872843425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,1536,0.004690133531888326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,1536,0.015638400117556253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,1536,0.006502399841944377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,1024,0.004036266605059306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,1024,0.015153066317240397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,1024,0.0056757330894470215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,768,0.003706666578849157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,768,0.015084800124168397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,768,0.005452799797058106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,512,0.003324799984693527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,512,0.014929067095120749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,256,0.003054933249950409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,256,0.014613333344459533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,5120,0.011043199896812439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,128,0.0027935999135176343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,128,0.014602667093276978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,65536,0.08338560263315836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,128,0.004705066482226053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,64,0.0026378666361172995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,64,0.014485333363215128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1536,32,0.002661333233118057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,16384,0.024791467189788818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1536,32,0.01455893317858378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,65536,0.04993600050608317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,12288,0.0199455996354421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,65536,0.03989013433456421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,51200,0.040009601910909014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,51200,0.034918399651845296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,16384,0.015953066945075988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,16384,0.021537067492802937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,8192,0.014959999918937683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,12288,0.013278933366139731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,12288,0.01990399956703186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,3584,0.00893440047899882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,10240,0.012050132950146992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,10240,0.01957546671231588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,10240,0.017321600516637167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,8192,0.010407466689745586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,8192,0.018557866414388023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,8192,0.014907733599344889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,7168,0.009734400113423665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,7168,0.017886932690938315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,7168,0.013274666666984559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,6144,0.008719999591509502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,6144,0.017386666933695474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,6144,0.011680000027020772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,512,0.005138133466243744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,5120,0.008400000135103862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,5120,0.010487467050552368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,5120,0.017959467569986978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,4096,0.00754559983809789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1536,256,0.004934399823347728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,4096,0.0173962672551473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,3584,0.007099733253320058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,3584,0.00886826713879903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,3584,0.016838399569193523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,3072,0.006487466891606649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,3072,0.016475733121236166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,3072,0.008060800035794576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,2560,0.006128000219662985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,2560,0.016266666849454246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,2048,0.005458133419354757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,2048,0.015979733069737753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,2048,0.006796800096829732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,1536,0.004711466530958811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,1536,0.015581867098808289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,1536,0.006366933385531108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,1024,0.00402453343073527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,1024,0.015343999862670899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,1024,0.005810133119424184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,768,0.0036202666660149893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,768,0.014967466394106546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,512,0.003390933324893316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,768,0.005413333574930826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,512,0.005094400048255921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,512,0.014773333072662353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,256,0.0030080000559488933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,65536,0.08377280235290527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,256,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,256,0.004828799764315287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,128,0.002828799933195114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,128,0.0049685334165891016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,64,0.003094399968783061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,128,0.014833066860834757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,1024,32,0.003068800022204717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,64,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,1024,32,0.014879999558130899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,51200,0.06660266717274985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,65536,0.03905066649119059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,51200,0.03176640073458354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,16384,0.024513065814971924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,51200,0.030705066521962483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,16384,0.013331199685732523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,51200,0.06633066733678182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,16384,0.020131200551986694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,12288,0.011475200454394024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,12288,0.01899626652399699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,12288,0.019612799088160195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,10240,0.010221866766611735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,12288,0.01985493302345276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,10240,0.018782933553059898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,10240,0.01714986761411031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,8192,0.0145578662554423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,8192,0.01730560064315796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,7168,0.008545066912968953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,7168,0.017972266674041747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,7168,0.013529599706331889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,6144,0.007831466694672901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,6144,0.017423999309539796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,5120,0.00732479989528656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,5120,0.018023467063903807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,65536,0.03517226775487264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,65536,0.08437013626098633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,4096,0.01722559928894043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,4096,0.009413333733876546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,3584,0.006620799998442332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,8192,0.009213866790135701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,4096,0.009246933460235595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,3584,0.017220266660054526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,3072,0.006018133461475372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,3072,0.016680532693862916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,3072,0.008086400230725606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,2560,0.006090666850407918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,1024,2560,0.0075914666056633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,2560,0.0076906666159629825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,2048,0.005357866485913595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,2048,0.015659733613332113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,4096,0.006202666461467743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,1536,0.005066666503747304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,1536,0.015511467059453329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,1536,0.006552533308664958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,1024,0.0040394666294256846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,1024,0.015264000495274863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,16384,0.02453440030415853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,768,0.003643733263015747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,1024,0.0056757330894470215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,768,0.00535999983549118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,768,0.015237333377202353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,512,0.003402666747570038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,512,0.005076266825199127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,512,0.014963199694951376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,256,0.0031658666829268134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,2560,0.016160000363985697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,256,0.014660267035166421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,128,0.0030165334542592366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,128,0.004668800036112467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,128,0.014636799693107605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,64,0.00279573326309522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,768,32,0.002834133307139079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,64,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,6144,0.011738666892051696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,768,32,0.014599466323852539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,65536,0.027667200565338133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,65536,0.028380799293518066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,51200,0.022807466983795165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,51200,0.025831466913223265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,65536,0.0834122657775879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,16384,0.010475732882817586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,16384,0.018833067019780478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,16384,0.0243285338083903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,12288,0.009131733576456707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,12288,0.017271467049916587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,12288,0.01924906571706136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,10240,0.008583466211954754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,10240,0.018004266421000163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,10240,0.016709333658218382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,8192,0.007853866616884867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,8192,0.017607466379801432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,8192,0.014097066720326743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,7168,0.006677333513895671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,7168,0.018388267358144125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,7168,0.01291306714216868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,6144,0.006182399888833364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,6144,0.017497599124908447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,6144,0.01157866617043813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,5120,0.006382933259010315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,5120,0.018091734250386557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,5120,0.010406399766604107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,4096,0.00594346672296524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,4096,0.017552000284194947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,3584,0.006442666550477346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,3584,0.017030400037765504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,3584,0.008961066603660583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,3072,0.006088533500830332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,256,0.004859733581542969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,3072,0.016613333423932394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,2560,0.006205866734186808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,2560,0.007606400052706401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,2560,0.016216533382733662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,2048,0.005423999826113383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,2048,0.015982932845751443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,2048,0.006796800096829732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,1536,0.0047541335225105286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,1536,0.015637333194414772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,1536,0.006478933493296306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,5120,0.010554666320482891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,1024,0.004177066683769226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,768,0.003818666686614355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,1024,0.015455999970436096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,1024,0.005614933371543884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,512,0.0034901333351929987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,768,0.005415466427803039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,768,0.015215999881426492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,256,0.0030965333183606463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,512,0.004969599843025208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,512,0.014869333306948344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,768,2048,0.006810666620731353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,128,0.002899199972550074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,256,0.014685866236686707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,128,0.0142794668674469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,64,0.01437333325544993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,32,0.002681600054105123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,512,32,0.0143477330605189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,65536,0.017089066902796428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,65536,0.023081600666046143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,65536,0.08303146362304688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,3584,0.008920533458391826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,51200,0.01447466711203257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,51200,0.021575466791788737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,256,0.004828799764315287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,16384,0.007572266459465027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,512,64,0.002696533252795537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,51200,0.06578133503595987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,16384,0.017844265699386595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,16384,0.023970133066177367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,12288,0.0173418660958608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,10240,0.006387199958165486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,10240,0.01790720025698344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,10240,0.01639359990755717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,8192,0.006154666841030121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,8192,0.017283199230829875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,8192,0.014114133516947427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,7168,0.005983999868233999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,7168,0.018151466051737467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,7168,0.012872533003489176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,6144,0.005906133353710175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,6144,0.017588265736897788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,4096,0.009282132983207703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,5120,0.006267733375231425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,5120,0.01795733372370402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,3072,0.008024533092975617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,4096,0.005845333139101664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,4096,0.01764586567878723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,4096,0.009193600217501322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,3584,0.006357333560784657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,512,128,0.004766933123270671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,12288,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,3584,0.008833066622416178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,3072,0.005967999994754791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,3072,0.0165173331896464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,3072,0.007996800045172375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,2560,0.006120533247788747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,2560,0.01609386702378591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,2048,0.005386666456858317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,2048,0.015824000040690102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,1536,0.004655999938646952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,1536,0.015355733036994935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,1536,0.006333866715431213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,1024,0.0039434666434923805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,51200,0.06629120111465454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,1024,0.015030399958292643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,768,0.0035968000690142312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,768,0.014859732985496522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,3584,0.016846932967503867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,768,0.005307733515898387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,512,0.003369600077470144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,512,0.014636799693107605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,512,0.005060266455014547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,256,0.0030229332546393077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,256,0.014518400033315023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,128,0.0028437333802382152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,128,0.014542933305104574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,128,0.004665599763393402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,64,0.0026335999369621276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,12288,0.01887680093447367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,256,32,0.0026208000878492994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,64,0.014504533012708029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,65536,0.011638399958610535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,256,32,0.014473600188891092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,65536,0.01986879905064901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,51200,0.010100266337394715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,51200,0.0185205340385437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,51200,0.0654581348101298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,16384,0.005962666869163513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,16384,0.01753066579500834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,6144,0.011603200435638427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,16384,0.02370026707649231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,12288,0.005860266586144766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,12288,0.01771519978841146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,12288,0.018918399016062418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,10240,0.006089599927266439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,10240,0.017816533644994102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,10240,0.016537599762280784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,5120,0.010484266281127929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,8192,0.0059690664211908976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,7168,0.005810133119424184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,8192,0.01757226586341858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,8192,0.013967999815940857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,7168,0.01272213359673818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,6144,0.005670399963855743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,7168,0.01759679913520813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,5120,0.005995733539263407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,6144,0.01730666756629944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,256,0.0047872001926104225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,5120,0.010412800312042236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,5120,0.017646932601928712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,4096,0.00573333352804184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,4096,0.009155199925104777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,4096,0.01722986698150635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,3584,0.0061365331212679545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,3072,0.005791999896367391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,3584,0.008711466193199157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,3584,0.01685439944267273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,2560,0.005681066711743673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,3072,0.007964799801508587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,3072,0.016291200121243795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,2560,0.007623466849327088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,2560,0.01589120030403137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,2048,0.005122133096059163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,2560,0.007656533519426982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,2048,0.015719466408093772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,1536,0.004509866734345754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,2048,0.006820266445477803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,1536,0.015346133708953857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,65536,0.08294933636983236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,1024,0.003870933254559835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,1024,0.015105066696802774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,768,0.014855466286341348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,256,1024,0.005679999788602194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,512,0.0033589333295822145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,512,0.014756266276041666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,512,0.005026133358478546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,256,0.0029535998900731405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,256,0.01455893317858378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,2048,0.006685866912206013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,128,0.002844800055027008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,128,0.01444906691710154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,768,0.0035829332967599234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,64,0.0026133333643277483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,64,0.014506666858990987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,128,32,0.0026133333643277483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,128,32,0.014437333742777506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,65536,0.008663466572761536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,65536,0.018720000982284546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,51200,0.007479466497898102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,51200,0.018744534254074095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,16384,0.005776000022888183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,16384,0.01737706661224365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,12288,0.005682133138179779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,12288,0.01740586757659912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,10240,0.005895466605822245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,10240,0.017308799425760905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,6144,0.0116074671347936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,8192,0.005838933090368906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,7168,0.005656533439954122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,8192,0.017040000359217326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,6144,0.005532800157864889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,7168,0.01730026602745056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,5120,0.005907199780146281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,6144,0.01713706652323405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,4096,0.005538133283456167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,5120,0.017520000537236534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,3584,0.005994666616121928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,4096,0.016857600212097167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,3072,0.005657599866390228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,3584,0.016547200083732606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,3072,0.01628373364607493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,2560,0.0056415999929110205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,2560,0.015819733341534935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,2048,0.0051018665234247845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,1536,0.004488533238569895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,2048,0.015507200360298156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,1536,0.015238400300343832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,1024,0.00382080003619194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,768,0.003566933423280716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,1024,0.015030399958292643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,512,0.0032511999209721885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,768,0.014817066987355552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,256,0.0029866665601730345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,512,0.01497066617012024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,128,0.002765866617361705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,256,0.014404267072677612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,64,0.0026922665536403658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,64,32,0.002644266684850057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,128,0.01455573340257009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,65536,0.007925333579381307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,32,0.014403200149536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,51200,0.007606400052706401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,65536,0.01909439961115519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,16384,0.005869866907596588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,51200,0.018694400787353516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,12288,0.005762133498986562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,16384,0.017488000790278117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,10240,0.005871999760468801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,12288,0.01742080052693685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,8192,0.005781333148479462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,10240,0.017388800779978432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,7168,0.005684266487757364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,8192,0.01692906618118286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,6144,0.005515733361244201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,7168,0.017448532581329345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,6144,0.0170741339524587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,5120,0.005864533285299936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,4096,0.005524266759554545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,5120,0.017523199319839478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,3584,0.0059690664211908976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,4096,0.01688213348388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,3072,0.0056426664193471275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,1536,0.006341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,3584,0.01664959987004598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,2560,0.005724800129731497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,3072,0.01627946694691976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,2048,0.005096533397833506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,2560,0.015966932972272238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,1536,0.004477866490681966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,1024,0.0037973334391911825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,1024,0.005602133274078369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,1536,0.015196800231933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,128,0.004675200084845225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,768,0.0052821333209673565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,1024,0.014966400464375815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,512,0.003159466634194056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,768,0.014684800306955972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,512,0.01479573349157969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,256,0.002882133424282074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,128,0.002657066782315572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,256,0.014349866906801859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,128,0.01420799990495046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,64,0.002621866762638092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,32,0.0025909334421157838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,64,0.014299733440081277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,32,0.014371200402577718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,7,128,256,0.004855466882387797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,32,2048,0.015576533476511636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,7,32,768,0.003550933301448822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,7,64,64,0.014551466703414917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,16384,0.3914090792338053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,16384,0.7469429016113281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,16384,0.3561621348063151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,51200,1.0915157318115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,51200,1.1796704610188802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,12288,0.5697226842244466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,12288,0.27251628239949544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,12288,0.29728320439656575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,10240,0.47594238917032883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,10240,0.2632160027821859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,10240,0.2918602625528971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,8192,0.38135251998901365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,8192,0.20474559466044107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,51200,2.3377151489257812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,7168,0.32914559046427405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,7168,0.1586858590443929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,7168,0.17962452570597331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,6144,0.15674667358398436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,6144,0.28689600626627604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,5120,0.13309120337168376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,6144,0.15982826550801593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,4096,0.19993066787719727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,3584,0.16756374041239422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,3584,0.08269440333048503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,3584,0.11263466676076253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,3072,0.08640639781951905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,3072,0.14413973490397136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,2560,0.0749141295750936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,2560,0.06123520135879516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,2048,0.09671573638916016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,2048,0.06278293530146281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,2048,0.050366934140523276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,1536,0.07433386643727621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,1536,0.05156373182932535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,5120,0.2371061325073242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,1024,0.05890880028406779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,1024,0.03927893241246541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,1024,0.028613332907358808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,768,0.03941333293914795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,768,0.03351680040359497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,2560,0.12091946601867676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,512,0.03107200066248576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,512,0.026964267094930012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,512,0.016833066940307617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,256,0.015197867155075073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,256,0.020745599269866945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,256,0.011975466211636861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,8192,0.17999359766642253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,128,0.008992000420888265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,128,0.009239466985066731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,64,0.005983999868233999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,128,0.020898133516311646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,65536,32,0.004474666714668274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,64,0.01858773430188497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,4096,0.10852373441060384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,65536,32,0.01876693367958069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,5120,0.13168426354726154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,4096,0.10841493606567383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,3072,0.0717461347579956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,1536,0.039477332433064775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,51200,0.9652544021606445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,65536,1.2160757700602214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,65536,768,0.022936532894770302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,51200,0.9032522837320963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,16384,0.6025962829589844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,51200,1.8626944224039714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,16384,0.35225067138671873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,16384,0.3442229270935059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,12288,0.45736001332600906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,65536,2.3648427327473955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,12288,0.2221984068552653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,10240,0.20018347104390463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,12288,0.2533877372741699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,8192,0.3077301343282064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,10240,0.1863584041595459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,8192,0.18771306673685711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,10240,0.43751678466796873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,7168,0.14434879620869953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,8192,0.15365333557128907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,7168,0.13363946278889974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,6144,0.2253589312235514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,6144,0.14398080507914227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,5120,0.1872927983601888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,5120,0.11109973589579265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,4096,0.09344853560129801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,4096,0.17631999651590985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,3584,0.13232320149739582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,3584,0.07910186449686686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,3584,0.06946132977803549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,3072,0.11409920056660969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,3072,0.07171626885732016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,3072,0.06032533248265585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,2560,0.09569173653920492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,2560,0.06104000012079874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,2560,0.0516053318977356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,2048,0.05220906734466553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,7168,0.2641610622406006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,2048,0.0421994686126709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,65536,1.1522421518961588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,1536,0.06724159717559815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,1536,0.04276906649271647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,5120,0.10750933488210042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,1024,0.04494719902674357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,1024,0.03377600113550822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,1024,0.024819199244181314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,1536,0.03746773401896159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,768,0.03181866606076558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,768,0.028699733813603717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,512,0.02453226645787557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,512,0.023887999852498374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,256,0.013481600085894265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,256,0.01930560072263082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,128,0.007628799974918365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,256,0.01090773344039917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,128,0.015500799814860026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,128,0.007985066870848339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,64,0.0041802664597829185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,32,0.004082133372624716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,64,0.015681067109107973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,51200,32,0.015752533078193666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,51200,2048,0.07709120114644369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,65536,0.38229118982950844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,6144,0.1324501355489095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,65536,0.7400778452555339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,4096,0.07827946345011393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,51200,0.29898134867350257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,16384,0.1071402629216512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,51200,0.30222934087117515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,16384,0.1845952033996582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,51200,0.5870005289713542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,12288,0.14035733540852863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,12288,0.09688320159912109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,12288,0.07215893268585205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,10240,0.07261760234832763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,10240,0.13798294067382813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,8192,0.09441386858622233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,10240,0.06069973309834799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,8192,0.061483732859293615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,8192,0.05299839973449707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,7168,0.08265706698099771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,7168,0.05576320091883341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,7168,0.04427413145701091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,6144,0.07234666347503663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,6144,0.04992213249206543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,5120,0.06020053227742513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,5120,0.044197332859039304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,4096,0.04906560182571411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,4096,0.03796906471252441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,3584,0.04348053137461345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,4096,0.03171520034472148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,3584,0.035382401943206784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,3584,0.025177599986394246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,768,0.01998293399810791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,3072,0.037701332569122316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,3072,0.032757333914438885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,51200,512,0.015069866180419922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,3072,0.022310400009155275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,2560,0.029743999242782593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,2560,0.019789866606394448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,2048,0.02606826623280843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,1536,0.02062186598777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,2048,0.016461867094039916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,2048,0.02650986711184184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,1024,0.014601600170135499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,1536,0.023500800132751465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,1024,0.010681600371996561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,1024,0.020436267058054604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,768,0.011550933122634888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,768,0.0191648006439209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,768,0.009272533655166625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,512,0.008570667107899983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,512,0.01734293301900228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,256,0.005764266848564148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,256,0.015211733182271323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,256,0.005433600147565206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,128,0.0035306667288144433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,128,0.014833066860834757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,128,0.005062399804592133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,64,0.003223466624816259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,64,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,2560,0.031972267230351764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,16384,32,0.0032085334261258446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,65536,0.35527467727661133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,16384,32,0.014763733744621277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,1536,0.013548800349235534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,16384,0.09421333471934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,65536,0.28349758783976237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,65536,0.2962186813354492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,65536,0.5526336034138997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,16384,0.16933333079020182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,51200,0.4346666653951009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,51200,0.2340768019358317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,6144,0.03882559935251872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,5120,0.03350079854329427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,12288,0.11005333264668782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,16384,0.08746559619903564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,12288,0.06698666413625082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,12288,0.05740693410237631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,10240,0.05845866600672404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,10240,0.09240319728851318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,8192,0.07553386688232422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,8192,0.04937493403752645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,7168,0.06643733183542887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,7168,0.04510506788889567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,6144,0.05496213436126709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,6144,0.04113279978434245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,6144,0.03192533254623413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,5120,0.045365333557128906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,16384,512,0.007605333129564922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,5120,0.03701119820276896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,5120,0.02739199995994568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,4096,0.03678613503774007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,4096,0.032664533456166586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,4096,0.022873600323994957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,3584,0.03335893154144287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,3584,0.021086933215459187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,3584,0.030424533287684123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,3072,0.02878933350245158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,3072,0.028076799710591634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,3072,0.01873706579208374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,2560,0.024259199698766075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,2560,0.02555946707725525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,2048,0.02017813324928284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,2048,0.023421865701675416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,1536,0.015947733322779337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,1536,0.02145919998486837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,1536,0.011800533533096314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,1024,0.011659733454386393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,1024,0.01921280026435852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,1024,0.009357866644859315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,768,0.009790933132171631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,768,0.01825066606203715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,51200,0.21594559351603188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,512,0.008190933366616566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,16384,0.0874720017115275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,512,0.015377066532770791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,256,0.004033066580692927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,512,0.005926399926344554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,10240,0.04877866506576538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,256,0.015076266725858054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,256,0.005243733525276184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,128,0.0035103999078273775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,64,0.0032320000231266023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,128,0.014857600132624308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,8192,0.04033066829045613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,64,0.014801067113876343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,7168,0.03574613332748413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,12288,32,0.014891733725865683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,65536,0.23655254046122232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,65536,0.2510805288950602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,65536,0.4754602750142415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,2560,0.016556800405184428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,51200,0.20111893018086752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,51200,0.1853343963623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,51200,0.3733279863993327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,2048,0.013684266805648803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,16384,0.1247104008992513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,16384,0.07613973617553711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,12288,0.059322667121887204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,12288,0.09846719900767008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,12288,32,0.0031445334355036415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,10240,0.07953066825866699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,10240,0.051666132609049474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,10240,0.041332264741261796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,8192,0.06654293139775594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,8192,0.03519146839777629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,7168,0.05366506576538086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,7168,0.03113173246383667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,7168,0.04166293144226074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,6144,0.04697706699371338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,6144,0.027163734038670857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,5120,0.03941546678543091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,768,0.008250666658083598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,5120,0.03420693476994832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,4096,0.03191679914792379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,4096,0.029963733752568562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,4096,0.020523732900619505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,3584,0.029338665803273517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,3584,0.027668267488479614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,3584,0.018490666151046754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,3072,0.024898133675257363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,3072,0.025623466571172076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,12288,0.04892266591389974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,2560,0.021171200275421142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,2560,0.02396906614303589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,8192,0.043891199429829914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,2560,0.014656000336011252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,2048,0.01737173398335775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,2048,0.012659200032552085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,2048,0.022309333086013794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,1536,0.013713066776593527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,1536,0.020578134059906005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,1024,0.010060800115267436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,1024,0.018598399559656777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,1024,0.00879253347714742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,768,0.008483200271924337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,6144,0.03722453514734904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,768,0.017107200622558594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,768,0.007063466807206471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,512,0.01509119967619578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,512,0.005686399837334951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,256,0.0035775999228159585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,256,0.015069866180419922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,256,0.005240533252557119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,128,0.0032298666735490165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,128,0.014843733112017313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,128,0.004816000163555145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,64,0.002946133414904277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,64,0.014877866705258688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,16384,0.06684799989064535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,32,0.0029493334392706556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,12288,128,0.0048767998814582825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,10240,32,0.014723199605941772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,65536,0.1997375965118408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,65536,0.37377068201700847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,65536,0.18434240023295084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,10240,512,0.006673066814740498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,16384,0.09215573469797769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,51200,0.1601909319559733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,51200,0.14528212547302247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,5120,0.023881600300470988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,51200,0.29543466567993165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,16384,0.06061866680781046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,12288,0.07044906616210937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,16384,0.05094293355941772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,10240,0.05878506501515707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,12288,0.04967999855677287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,12288,0.04063786665598552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,10240,0.04380266666412354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,10240,0.034126933415730795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,8192,0.028522666295369464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,7168,0.025550933678944905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,8192,0.03917866547902425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,7168,0.03503679831822713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,3072,0.016577066977818807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,6144,0.03717120091120402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,5120,0.03136746684710185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,6144,0.022703999280929567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,6144,0.03240320086479187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,5120,0.02959146698315938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,4096,0.025563732782999678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,5120,0.019935999313990274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,3584,0.022920533021291097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,4096,0.026359466711680095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,3584,0.015672533710797628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,3072,0.023269333442052207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,3072,0.013722667098045349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,2560,0.017309866348902383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,2560,0.012652800480524699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,2560,0.022035199403762817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,2048,0.014221866925557455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,2048,0.020578134059906005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,1536,0.011560533444086711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,10240,1536,0.010687999924023946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,1536,0.019014400243759156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,1024,0.008513066172599792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,1536,0.009724799791971843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,1024,0.007859200239181519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,1024,0.017129600048065186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,768,0.007322666545708974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,768,0.014978133638699851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,768,0.0060597335298856105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,512,0.005564799904823304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,512,0.015277866522471109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,512,0.005623466769854228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,256,0.0034965333839257562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,256,0.014787200093269347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,128,0.00315733328461647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,128,0.014575999975204468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,8192,0.04798933267593384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,128,0.004780800143877665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,64,0.002976000060637792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,64,0.014380799730618796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,32,0.0029578665892283124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,32,0.014526933431625366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,3072,0.020151466131210327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,65536,0.3262399991353353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,65536,0.18242239952087402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,65536,0.16315414110819498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,51200,0.1472277323404948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,51200,0.25751999219258626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,16384,0.10239253044128419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,16384,0.05630613168080648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,16384,0.04565013249715169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,12288,0.06588053305943807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,8192,7168,0.04239039818445842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,12288,0.04717973470687866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,8192,3584,0.02490773399670919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,10240,0.054523734251658115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,10240,0.0412831981976827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,8192,0.04475520054499309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,4096,0.01739733417828878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,8192,0.03532266616821289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,7168,0.039800532658894855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,7168,0.032994133234024045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,2048,0.01112320025761922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,7168,0.023166932662328086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,6144,0.03531519969304402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,6144,0.03063360055287679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,5120,0.03001813292503357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,5120,0.0280458668867747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,8192,256,0.005053866902987162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,4096,0.023191465934117635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,4096,0.024995199839274087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,51200,0.12929493586222332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,3584,0.02071466644605001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,3584,0.02408533294995626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,3072,0.018267732858657838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,3584,0.01427839994430542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,3072,0.022553600867589316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,2560,0.015782399972279867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,2560,0.021309866507848104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,2560,0.01183573305606842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,2048,0.013300266861915589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,2048,0.02009920080502828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,2048,0.010319999853769938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,1536,0.010827733079592387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,12288,0.03595626751581828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,1536,0.009139200051625569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,1536,0.01878933310508728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,1024,0.00804373323917389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,1024,0.015315199891726175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,10240,0.03067306677500407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,768,0.015195733308792115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,8192,0.025783467292785644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,512,0.004358399907747904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,512,0.015319466590881348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,512,0.005594666798909505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,6144,0.020861866076787312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,256,0.003337600082159042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,256,0.01487573285897573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,256,0.0051818668842315676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,128,0.003049599876006444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,128,0.014775466918945313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,128,0.004811733464399974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,5120,0.018088533480962118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,64,0.01456000010172526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,32,0.0029258665939172106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,7168,32,0.014636799693107605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,65536,0.15640320777893066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,65536,0.2798240025838216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,4096,0.015576533476511636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,65536,0.14260692596435548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,51200,0.2214954694112142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,51200,0.12611200014750162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,51200,0.11372693379720052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,16384,0.08760853608449301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,16384,0.05156160195668539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,16384,0.040350933869679764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,12288,0.05758506854375204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,12288,0.04238933324813843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,10240,0.04938666820526123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,10240,0.037638401985168456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,10240,0.027432533105214436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,8192,0.0395359992980957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,64,0.002918400118748347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,8192,0.03378986517588298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,8192,0.022884267568588256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,7168,0.03464959859848023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,7168,0.031430399417877196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,7168,0.02073600093523661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,6144,0.0301749328772227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,6144,0.029051733016967774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,5120,0.0254528005917867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,6144,0.018683733542760213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,4096,0.021359999974568687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,5120,0.026708267132441205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,4096,0.024387200673421226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,4096,0.013848533233006796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,3072,0.012703999876976013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,3584,0.01904426614443461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,3584,0.023156267404556275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,3584,0.01308799982070923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,3072,0.016876800855000814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,2560,0.014616533120473226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,3072,0.011762133240699768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,3072,0.021964800357818604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,2560,0.010867200295130412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,2048,0.019324799378712974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,7168,768,0.006897066533565521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,2048,0.009502933422724406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,1536,0.010003200173377991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,1536,0.018309332927068076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,1536,0.008874666690826417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,1024,0.007500799993673961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,1024,0.006313600142796834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,1024,0.015479466319084168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,1024,0.005948799848556519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,768,0.006405333181222279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,768,0.00574186642964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,768,0.015332266688346863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,512,0.003822933385769526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,512,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,512,0.005363200108210246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,256,0.003336533407370249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,256,0.014903466900189719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,256,0.0049898669123649595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,128,0.0030239999294281008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,128,0.014599466323852539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,64,0.002825599908828735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,64,0.014562132954597472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,32,0.002825599908828735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,32,0.01452906628449758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,6144,2560,0.02050666610399882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,65536,0.23837760289510093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,65536,0.13714240392049154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,65536,0.12352426846822102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,51200,0.10745920340220134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,51200,0.18847999572753907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,7168,768,0.006078933179378509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,16384,0.045102934042612716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,16384,0.06617706616719564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,51200,0.09853759606679281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,12288,0.03236693342526754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,12288,0.048792533079783124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,12288,0.03745493491490682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,12288,0.02850773334503174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,10240,0.04154880046844482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,10240,0.03347733418146769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,10240,0.024474666515986124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,8192,0.03367893298467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,8192,0.020616533358891805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,8192,0.029921066761016846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,7168,0.01859626571337382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,7168,0.028161066770553588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,6144,0.025135999917984007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,6144,0.02608319918314616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,6144,0.01660053332646688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,5120,0.02114026745160421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,6144,2048,0.012358400225639343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,5120,0.016383999586105348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,5120,0.02385279933611552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,4096,0.017568000157674155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,4096,0.02245546579360962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,3584,0.015962666273117064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,3584,0.012193066875139873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,3584,0.02166293263435364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,3072,0.014058666427930197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,3072,0.020470400651295982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,3072,0.010875733693440755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,2560,0.012116266290346782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,2560,0.019884800910949706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,2048,0.010497066378593444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,2048,0.018745599190394084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,2048,0.009198932846387228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,1536,0.008733866612116496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,1536,0.0076682666937510175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,7168,0.028076799710591634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,1024,0.006844800213972728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,1024,0.0063296000162760425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,6144,128,0.0047872001926104225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,1024,0.015383467078208923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,768,0.005420800050099691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,768,0.00582826683918635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,512,0.0036458666125933326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,768,0.015218133727709452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,256,0.003270400067170461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,512,0.005447466671466827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,512,0.014907733599344889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,128,0.002998399982849757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,256,0.005060266455014547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,256,0.01469013293584188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,64,0.002844800055027008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,128,0.014541866381963095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,64,0.014622933665911355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,5120,32,0.0027797333896160126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,1536,0.0170741339524587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,5120,32,0.014643200238545737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,16384,0.03543999989827474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,128,0.005049600203831991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,65536,0.18633599281311036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,65536,0.10891306400299072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,65536,0.10643946329752604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,5120,0.014563199877738953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,4096,0.012666666507720947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,51200,0.08893547058105469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,51200,0.1476042588551839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,51200,0.08519039948781332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,16384,0.0309173325697581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,16384,0.04988799889882405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,12288,0.033353598912556966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,5120,2560,0.010362666845321656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,12288,0.03892373243967692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,10240,0.030461867650349934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,10240,0.021683200200398763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,8192,0.0180021325747172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,8192,0.02693973382314046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,7168,0.024741333723068238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,6144,0.02203413248062134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,7168,0.025345067183176678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,6144,0.01487573285897573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,6144,0.023860265811284385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,5120,0.01911466717720032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,5120,0.02257386644681295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,5120,0.013306666413942972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,4096,0.016126933693885803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,4096,0.020891733964284263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,4096,0.011682132879892986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,3584,0.013170133034388224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,3584,0.020141865809758505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,3584,0.011079466342926026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,3072,0.011801600456237793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,3072,0.01949440042177836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,3072,0.01034453312555949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,2560,0.010307199756304423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,2560,0.009645866354306538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,2560,0.018807466824849448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,2048,0.008964266379674275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,2048,0.017926400899887084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,1536,0.007590400179227193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,16384,0.03888746500015259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,12288,0.025151999791463216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,10240,0.03301333387692769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,8192,0.027422932783762614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,1536,0.015378133455912272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,1024,0.0060928001999855045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,1536,0.007004799942175548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,1024,0.01530346671740214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,768,0.0039434666434923805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,768,0.015019733707110086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,768,0.005569066603978475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,512,0.003537066777547201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,256,0.003142400085926056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,512,0.005243733525276184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,512,0.015068800250689188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,256,0.014760532975196838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,256,0.004939733445644379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,128,0.014452266693115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,128,0.004749866823355356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,32,0.002799999962250392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,64,0.014314666390419006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,4096,32,0.014432000120480857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,65536,0.1682773272196452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,65536,0.10061439673105876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,51200,0.13083626429239908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,51200,0.08318506876627604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,2048,0.008252800007661184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,51200,0.08181546529134115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,16384,0.047687466939290366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,16384,0.036449066797892254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,16384,0.029782400528589888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,12288,0.03487679958343506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,1024,0.006009600063165029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,12288,0.03110186656316121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,12288,0.02400426665941874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,10240,0.029942399263381957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,10240,0.021081600586573282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,10240,0.028705066442489623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,8192,0.024642133712768556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,128,0.0028917332490285236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,7168,0.022410666942596434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,4096,64,0.0028138667345046996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,8192,0.025041067600250246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,7168,0.024105600516001382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,7168,0.01615466674168905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,6144,0.019732266664505005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,6144,0.0229312002658844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,4096,7168,0.016302933295567833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,5120,0.022050132354100548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,4096,0.013396267096201578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,5120,0.013089066743850708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,3584,0.012037332852681477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,4096,0.011113599936167399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,4096,0.020360533396402994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,3072,0.010827733079592387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,3584,0.019719467560450236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,3072,0.010066133737564088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,3072,0.019130667050679527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,2560,0.00960106650988261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,2560,0.01898026665051778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,2560,0.009382399916648864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,2048,0.008363733688990276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,1536,0.007111466427644093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,2048,0.0078015998005867004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,1536,0.006794666747252147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,2048,0.0161461333433787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,1024,0.0046634669105211895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,1024,0.01536853313446045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,768,0.003841066608826319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,768,0.015284267067909241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,8192,0.01765973369280497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,768,0.0057087997595469155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,512,0.003516799956560135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,3584,0.010553600390752156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,512,0.015043200055758158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,256,0.0031839999059836066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,512,0.005284266670544943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,256,0.004913066824277243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,256,0.014641066392262777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,128,0.002841600030660629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,128,0.004774400095144907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,64,0.0027744000156720476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,64,0.01440000037352244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,32,0.0027850667635599775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,32,0.014407466848691305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,65536,0.10300693511962891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,65536,0.14494080543518068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,1536,0.015662933389345803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,65536,0.08753493626912436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,51200,0.1137279987335205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,51200,0.07160853544871013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,51200,0.07517333030700683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,16384,0.039879465103149415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,16384,0.03370026747385661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3584,5120,0.0172650674978892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,16384,0.02789120078086853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,12288,0.03156373302141825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,12288,0.028843732674916585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,12288,0.02257066567738851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,10240,0.026547199487686156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,8192,0.02212266723314921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,10240,0.026554665962855023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,8192,0.025095466772715253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,7168,0.019822933276494346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3584,128,0.014620799819628397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,7168,0.023565866549809775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,6144,0.014566399653752646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,6144,0.017550933361053466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,7168,0.015362133582433065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,6144,0.013776000340779623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,6144,0.022281599044799805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,5120,0.012453333536783854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,4096,0.013797332843144735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,4096,0.01955839991569519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,4096,0.011085866888364156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,3584,0.012662399808565775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,3584,0.019066667556762694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,3072,0.011306666334470113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,3584,0.010538666447003683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3584,1024,0.0061471998691558834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,3072,0.018604799111684164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,2560,0.00902186632156372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,2560,0.00876800020535787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,2560,0.018038400014241538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,2048,0.00775786687930425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,2048,0.016350932916005454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,1536,0.006772266825040181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,1536,0.015889066457748412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,1536,0.0066101332505544026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,1024,0.004384000102678935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,5120,0.01544426679611206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,1024,0.015326933066050211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,5120,0.020971733331680297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,768,0.003918933371702829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,1024,0.006060799956321717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,768,0.015180800358454385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,768,0.005553066730499268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,512,0.005291733145713806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,512,0.014884266257286071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,256,0.003186133255561193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,256,0.014654933412869772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,256,0.0049450665712356566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,128,0.0029525332152843474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,128,0.004785066843032837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,128,0.014656000336011252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,64,0.002696533252795537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,32,0.0027274665733178455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,64,0.014598400394121806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,3072,32,0.014621866742769876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,65536,0.12013866901397705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,65536,0.07637440363566081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,65536,0.09336746533711751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,65536,0.09016533692677817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,51200,0.06132373412450155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,2048,0.00747626672188441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,16384,0.03333119948705037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,51200,0.06815360387166342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,12288,0.02621226708094279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,16384,0.030555733044942218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,16384,0.02752853234608968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,10240,0.022874667247136434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,12288,0.026375466585159303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,12288,0.021970132986704506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,10240,0.01790720025698344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,8192,0.019038933515548705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,10240,0.024940800666809083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,8192,0.016710400581359863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,7168,0.01732800006866455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,8192,0.016288000345230102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,8192,0.022614399592081703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,7168,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,7168,0.02276159922281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,3072,512,0.0035594666997591654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,6144,0.015307733416557312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,5120,0.01356160044670105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,6144,0.013713066776593527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,10240,0.019721599419911702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,5120,0.012345600128173827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,4096,0.010760533809661865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,4096,0.019025067488352455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,3584,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,3584,0.010314666231473287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,3584,0.01844373345375061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,51200,0.09561493396759033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,3072,0.010044800241788228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,3072,0.017509333292643228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,3072,0.009205333391825358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,2560,0.008414933085441589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,2560,0.015895467003186545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,2048,0.007214933137098948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,2048,0.0071936001380284624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,2048,0.01590079963207245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,1536,0.005649066468079885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,1536,0.015410133202870688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,1024,0.004167466859022776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,1024,0.015150933464368185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,1024,0.005989333490530649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,768,0.0037962667644023894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,768,0.014963199694951376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,768,0.005669333537419637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,512,0.00348693331082662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,512,0.014803199966748556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,512,0.005208533505598704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,6144,0.02100906570752462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,256,0.0031146667897701263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,5120,0.020198400815327963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,4096,0.01170133352279663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,256,0.01474133332570394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,128,0.002917333443959554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,256,0.005096533397833506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,64,0.0027615999182065325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,128,0.014693333705266317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,64,0.014325333635012307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2560,32,0.014492799838383993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,65536,0.0967082659403483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,2560,0.008354133367538452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,65536,0.062159999211629235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,65536,0.08548053105672201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,51200,0.07508160273234049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,51200,0.0523967981338501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,16384,0.02725653251012166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,16384,0.02723520000775655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,16384,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,12288,0.02176213264465332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,12288,0.02415999968846639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,10240,0.01895893414815267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,10240,0.022744532426198324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,3072,3072,0.009598933657010396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,10240,0.01790720025698344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,8192,0.0158869336048762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,8192,0.021358933051427206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,8192,0.015367466211318969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,7168,0.014378666877746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,7168,0.014144000411033631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,7168,0.02064746618270874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,6144,0.012964266538619994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,6144,0.020101332664489747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2560,32,0.00275093341867129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,5120,0.01165226697921753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,5120,0.019530665874481202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,4096,0.010213333368301391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,5120,0.011643733580907185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,4096,0.01016213297843933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,4096,0.018550399939219156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,3584,0.009559466441472372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,3584,0.017480534315109254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,3584,0.009418666362762451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,3072,0.008955732981363932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,3072,0.008390399813652038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,3072,0.016140799721082053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,2560,0.015966932972272238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,2048,0.0065098668138186145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,2048,0.015927466750144958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,2048,0.007077333331108093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,1536,0.004889599978923798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,1536,0.015446399648984274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,128,0.004749866823355356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,1024,0.0040394666294256846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,1024,0.015372799833615622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,1024,0.005685333410898844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,2560,0.007572266459465027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,768,0.003770666569471359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,768,0.005416533350944519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,768,0.015219199657440185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,512,0.0033439998825391137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,512,0.005151999990145365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,512,0.014749866724014283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,51200,0.06786879698435465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,256,0.0031061333914597826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,128,0.0028704000016053517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,256,0.014605866869290671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2560,1536,0.006697600086530049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,12288,0.02041920026143392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,128,0.0046165332198143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,64,0.0026848000784715016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,2048,32,0.0026389333109060925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,64,0.01455466647942861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,32,0.014525866508483887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,65536,0.07436266740163168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,65536,0.05194880167643229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,65536,0.08391040166219076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,51200,0.059453864892323814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,51200,0.04393279949824015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,6144,0.012948266665140786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,16384,0.02202026645342509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,51200,0.06663146813710531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,12288,0.017900800704956053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,16384,0.02480106751124064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,16384,0.024691200256347655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,12288,0.019990400473276774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,12288,0.022315732638041177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,10240,0.015474133690198264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,8192,0.013215999801953635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,10240,0.017398399114608765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,10240,0.021271467208862305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,7168,0.012238933642705282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,8192,0.01511146624883016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,8192,0.020232532421747842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,7168,0.013845333456993103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,6144,0.011136000355084736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,7168,0.019517866770426433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,5120,0.01018346647421519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,6144,0.012668800354003907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,6144,0.018837332725524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,4096,0.009037866195042928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,5120,0.011307733257611592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,5120,0.019146666924158732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,3584,0.00857919951279958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,4096,0.009418666362762451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,4096,0.017053866386413576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,3072,0.00804799993832906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,3584,0.009041066964467366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,3584,0.017052799463272095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,2560,0.0070250665148099255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,3072,0.016523733735084534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,2560,0.007548800110816956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,2560,0.01629866659641266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,2048,128,0.01455573340257009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,2048,0.005334400137265523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,1536,0.00470719983180364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,2048,0.015914666652679443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,1536,0.006422399977842967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,1536,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,1024,0.003999999910593033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,2560,0.0077461332082748415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,1024,0.015130666891733804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,1024,0.005769599974155426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,768,0.0036320000886917113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,768,0.005436799923578898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,768,0.014963199694951376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,512,0.005004799862702688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,512,0.014922666549682616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,256,0.0030346666773160298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,256,0.014513066411018372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,256,0.004886400202910105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,128,0.0028234665592511495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,128,0.014478933811187745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,128,0.004731733103593191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,1536,0.006372266511122386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,64,0.0027114666998386385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,32,0.0026911998788515727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,64,0.014510933558146158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1536,32,0.014361600081125895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,65536,0.0497599999109904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,65536,0.04002559979756673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,51200,0.039851733048756915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,65536,0.08303679625193278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,51200,0.034777601559956864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,16384,0.015925332903862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,16384,0.021771732966105142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,51200,0.0660042683283488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,12288,0.0132341335217158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,16384,0.024465066194534302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,12288,0.019869865973790486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,12288,0.01971413294474284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,10240,0.011920000116030376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,8192,0.010382933417956035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,10240,0.017406932512919106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,10240,0.01932800014813741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,7168,0.009734400113423665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,8192,0.014845866958300272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,8192,0.01837013363838196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,6144,0.008593066533406576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,7168,0.018118399381637573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1536,512,0.0033514666060606635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,2048,256,0.004965333143870035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,6144,0.017436800400416057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,5120,0.0082997332016627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,4096,0.007218133409818013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,4096,0.01729493339856466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,6144,0.011799466609954835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,5120,0.01789439916610718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,3072,0.008066133161385854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1536,2048,0.00682773341735204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,3584,0.017256534099578856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,3584,0.008922666311264038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,3072,0.0062943999965985615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,3072,0.016587733229001363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,3072,0.008148266871770223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,2560,0.006106666723887126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,2048,0.005422933399677277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,2560,0.007629866898059845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,2560,0.016323199868202208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,1536,0.004802133142948151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,2048,0.006849066913127899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,2048,0.01586133340994517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,1536,0.01551040013631185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,1024,0.004088533421357473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,1536,0.006495999793211619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,768,0.0036703998843828833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,1024,0.005674666663010915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,1024,0.015230933825174967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,512,0.0034304000437259674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,768,0.01492586632569631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,512,0.014889599879582724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,256,0.003054933249950409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,256,0.014629333217938741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,128,0.002890666574239731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,3584,0.006942933301130931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,128,0.004711466530958811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,7168,0.013455999890963235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,128,0.014769066373507181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,5120,0.010469333330790202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,32,0.003045333425203959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,1024,64,0.003044266750415166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,4096,0.00923413336277008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,64,0.014732799927393594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,1024,32,0.014942933122316995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,512,0.005167999863624572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,65536,0.03476800123850505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,51200,0.03183146715164185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,16384,0.013356799880663553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,51200,0.030771199862162275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,16384,0.02029119928677877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,12288,0.01145919958750407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,65536,0.08348800341288248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,10240,0.010435199737548828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,12288,0.01882773240407308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,51200,0.06645866632461547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,10240,0.018152532974878947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,8192,0.00925546685854594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,8192,0.01685546636581421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,7168,0.008515200018882752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,7168,0.01760853330294291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,65536,0.03913173278172811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,6144,0.007855999966462452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,6144,0.017742933829625447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,6144,0.011789866288503011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,5120,0.007396266857783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,768,0.005393066505591074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,5120,0.01804800033569336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,4096,0.006162133316198985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,5120,0.010533333818117777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,1024,256,0.004939733445644379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,4096,0.017398399114608765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,3584,0.0066101332505544026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,3584,0.017072000106175742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,3072,0.006080000102519989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,3584,0.008874666690826417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,3072,0.016658133268356322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,2560,0.006055466830730438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,2560,0.016406400005022685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,2560,0.007626666625340779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,2048,0.0053962667783101406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,2048,0.01597653329372406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,2048,0.006851199766000111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,1536,0.004783999919891357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,1536,0.01574186682701111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,1536,0.006443733473618825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,1024,0.00410453329483668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,1024,0.015282133221626281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,12288,0.019740800062815346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,16384,0.024473599592844644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,768,0.0037098666032155357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,768,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,512,0.0033439998825391137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,10240,0.017155200242996216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,512,0.014972800016403198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,512,0.005107200145721436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,256,0.0032106667757034303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,256,0.004811733464399974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,128,0.0029343999922275543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,128,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,128,0.004681600133577982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,64,0.0028266665836175283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,64,0.014457600315411887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,8192,0.01436906655629476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,768,32,0.0028351999819278715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,65536,0.027553067604700728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,32,0.014638933539390563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,7168,0.01291306714216868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,65536,0.02829119960467021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,51200,0.022807466983795165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,65536,0.08333226839701334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,51200,0.025234133005142212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,16384,0.010497066378593444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,51200,0.066102401415507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,16384,0.019118932882944743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,12288,0.009165866176287334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,16384,0.02434773246447245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,12288,0.01950826644897461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,12288,0.01729173262914022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,10240,0.008649599552154542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,10240,0.01758613387743632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,8192,0.007732266684373219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,8192,0.01715946594874064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,8192,0.013938132921854654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,7168,0.006663466493288676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,7168,0.01788053313891093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,768,256,0.014551466703414917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,7168,0.01279039978981018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,6144,0.006178133189678192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,6144,0.017577600479125977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,5120,0.006311466793219249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,5120,0.017991467316945394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,5120,0.010444800059000652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,4096,0.009187199672063192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,4096,0.005835733314355215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,4096,0.017375999689102174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,3584,0.0064074665307998655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,3584,0.017002665996551515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,3072,0.008154666423797608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,3584,0.008826667070388794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,3072,0.006056533257166544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,3072,0.016724266608556113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,3072,0.008016000191370647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,2560,0.00613973339398702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,2560,0.01639253298441569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,2560,0.007677866518497467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,2048,0.005502933263778686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,2048,0.01581760048866272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,2048,0.006923733154932658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,1536,0.004713599880536398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,1536,0.015625600020090738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,1024,0.004091733445723852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,1024,0.015446399648984274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,1024,0.005660800139109293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,768,0.0038890667259693147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,768,0.015030399958292643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,1024,0.005798399945100148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,512,0.003517866631348928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,768,768,0.005334400137265523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,512,0.014889599879582724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,512,0.005039999882380167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,256,0.0031914666295051576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,256,0.004743466774622599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,256,0.014867200454076131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,128,0.002842666705449422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,128,0.004926933348178864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,128,0.01462399959564209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,64,0.00276053324341774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,512,32,0.0026880001028378804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,4096,0.009152000149091084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,65536,0.017151999473571777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,32,0.014371200402577718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,65536,0.023154133558273317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,65536,0.08371626536051432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,51200,0.01423893372217814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,51200,0.02113706668217977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,51200,0.06536853313446045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,16384,0.007609599828720092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,16384,0.01773866613705953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,12288,0.006291200220584869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,12288,0.01760639945665995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,12288,0.01885333259900411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,10240,0.0062047998110453285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,10240,0.01781760056813558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,10240,0.01623466710249583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,8192,0.0061034664511680605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,8192,0.017334399620691936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,7168,0.006009600063165029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,7168,0.01742080052693685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,7168,0.012838400403658547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,6144,0.005841066439946493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,10240,0.016596266627311708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,768,0.005465599894523621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,6144,0.01160426636536916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,6144,0.017308799425760905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,5120,0.010356266299883525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,5120,0.017884800831476845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,4096,0.00580266664425532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,4096,0.017148800690968833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,4096,0.009206400314966837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,3584,0.0062730665008227035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,3584,0.017132800817489625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,3584,0.008785067001978557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,3072,0.005977599819501241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,512,64,0.014553599556287131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,6144,0.011596799890200297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,3072,0.007893333335717519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,2560,0.0060810665289560955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,3072,0.016343466440836587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,2048,0.005376000205675761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,2560,0.007584000130494435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,2560,0.016335999965667723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,1536,0.004634666442871094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,2048,0.015800533692042033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,1536,0.006434133152167003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,1536,0.01551040013631185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,1024,0.004001066585381826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,1024,0.015209600329399109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,1024,0.005640533566474914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,768,0.003626666714747747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,768,0.015007999539375306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,768,0.005238399902979533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,512,0.0032992000381151833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,512,0.014739200472831726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,512,0.005100800096988678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,256,0.0029493334392706556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,256,0.014486400286356607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,128,0.002855466554562251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,128,0.014573867122332254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,128,0.004625066618124644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,64,0.002665599932273229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,64,0.014450132846832275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,512,1536,0.006404266754786174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,32,0.0026410666604836782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,256,32,0.014401066303253173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,256,5120,0.0062047998110453285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,65536,0.019819732507069907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,65536,0.08273279666900635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,51200,0.010199466347694397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,51200,0.018637865781784058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,51200,0.06576639811197917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,16384,0.0059797331690788266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,16384,0.018053332964579262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,12288,0.005829333265622457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,12288,0.017299199104309083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,12288,0.018946133057276406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,10240,0.006064000229040781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,10240,0.01750719944636027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,16384,0.02388906677563985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,8192,0.00589333325624466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,8192,0.017180800437927246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,8192,0.014107732971509298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,7168,0.005820799867312113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,65536,0.011678933103879293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,7168,0.017400532960891724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,8192,0.01399893363316854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,7168,0.012778666615486146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,6144,0.0056426664193471275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,5120,0.0060127998391787205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,6144,0.011559466520945232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,6144,0.0170741339524587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,5120,0.010379733641942342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,4096,0.00556160012880961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,5120,0.017460266749064125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,3584,0.006031999985376993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,4096,0.009224533041318258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,4096,0.01695573329925537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,16384,0.023785599072774253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,3072,0.0058378666639328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,3584,0.016702934106191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,2560,0.005706666906674703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,3072,0.00795413355032603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,3072,0.016338133811950685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,2560,0.0076000000039736434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,2048,0.0051594664653142296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,2048,0.0068010667959849044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,2048,0.01560640037059784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,1536,0.004491733511288961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,1536,0.015411200126012168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,1536,0.006397866706053417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,1024,0.0038463999827702843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,1024,0.015239466230074564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,768,0.003621333340803782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,2048,0.006690133114655812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,768,0.01479573349157969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,512,0.0033333333830038703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,768,0.005310933291912079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,512,0.004997333387533823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,512,0.014859732985496522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,256,0.00297173336148262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,256,0.01443839967250824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,256,0.004737066725889841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,128,0.0028885332246621448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,128,0.014619732896486918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,128,0.004640000065167745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,64,0.0026762666801611584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,128,32,0.002699733277161916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,64,0.01434879998366038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,256,256,0.004789333542188009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,65536,0.00883733332157135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,65536,0.018760534127553304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,51200,0.007607466479142506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,16384,0.005691733459631602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,51200,0.018503467241923012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,12288,0.00565119981765747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,16384,0.01726613243420919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,10240,0.005884799857934316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,12288,0.017177599668502807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,10240,0.01733013391494751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,8192,0.005843199789524078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,7168,0.005716266731421152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,8192,0.017156267166137697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,6144,0.0055861334005991616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,7168,0.017595734198888144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,3584,0.008689066767692566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,5120,0.005987200140953064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,6144,0.01698346734046936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,4096,0.005564799904823304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,5120,0.017407999436060587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,3584,0.006095999975999197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,4096,0.01699413259824117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,2560,0.016088533401489257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,3072,0.005709866682688395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,3584,0.0165994664033254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,2560,0.005611733098824819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,3072,0.016886399189631144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,2560,0.015965867042541503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,1536,0.004481066763401031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,2048,0.015983999768892924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,1024,0.003852800031503042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,1536,0.015223466356595359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,768,0.0035071998834609987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,1024,0.014985600113868713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,512,0.003147733211517334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,768,0.014763733744621277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,256,0.0029311999678611755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,512,0.014713600277900696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,128,0.0027978666126728057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,256,0.0145087997118632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,64,0.0026922665536403658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,128,0.014351999759674073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,32,0.0026880001028378804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,64,0.014460800091425577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,65536,0.008062933385372163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,64,32,0.014320000012715658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,51200,0.007698133091131846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,65536,0.019139200448989868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,16384,0.0058783998092015585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,51200,0.01872106591860453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,10240,0.016459733247756958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,12288,0.005737600227197012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,10240,0.005883733431498209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,12288,0.01730239987373352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,8192,0.005781333148479462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,10240,0.01750826636950175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,7168,0.005644799768924713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,8192,0.017093332608540852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,6144,0.005576533575852713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,7168,0.01730666756629944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,5120,0.005898666878541311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,6144,0.016960000991821288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,4096,0.005500799914201101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,5120,0.017603200674057008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,128,32,0.014447999993960061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,3584,0.00597866674264272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,3072,0.005677866439024607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,3584,0.016492799917856852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,2560,0.0056202664971351625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,3072,0.0163263996442159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,2048,0.00498879998922348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,2560,0.015953066945075988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,1536,0.004457599918047587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,2048,0.01563093364238739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,1536,0.015290666619936624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,768,0.0034645333886146545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,1024,0.015034666657447815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,512,0.0032501332461833954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,768,0.014857600132624308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,256,0.002842666705449422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,512,0.014586666226387024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,128,0.002765866617361705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,256,0.014447999993960061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,64,0.002630399912595749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,128,0.014470400412877402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,32,0.0026538667579491934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,64,0.014286933342615762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,64,2048,0.004994133114814758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,32,0.014292266964912415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,6,128,1024,0.005765333275000254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,16384,0.017529600858688356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,6,32,4096,0.01702400048573812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,6,32,1024,0.003841066608826319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,16384,0.39146560033162436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,16384,0.3548341433207194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,16384,0.7537589391072591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,12288,0.5670442581176758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,12288,0.2968693415323893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,10240,0.24401386578877768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,51200,1.179746119181315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,10240,0.47307732899983723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,8192,0.3799807866414388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,8192,0.20616745948791504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,51200,1.09224001566569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,8192,0.2112511952718099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,7168,0.15802027384440104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,7168,0.32911574045817055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,7168,0.1799488067626953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,6144,0.33617919286092124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,6144,0.15609599749247233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,6144,0.13652373949686686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,5120,0.2369311968485514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,51200,2.336442565917969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,5120,0.137773863474528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,5120,0.1270645300547282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,10240,0.28020054499308267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,4096,0.1085205316543579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,3584,0.16712427139282227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,3584,0.09776000181833902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,4096,0.10800853570302327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,3072,0.08682986895243326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,3072,0.14427520434061686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,2560,0.14141012827555338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,2560,0.07479360103607177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,2560,0.061045332749684655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,2048,0.06266560157140097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,2048,0.05032000144322714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,1536,0.07445440292358399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,1536,0.05121493339538574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,1024,0.0504309336344401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,1024,0.038761599858601885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,1024,0.028357332944869994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,4096,0.19360960324605306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,768,0.03938666582107544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,768,0.03658026854197184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,12288,0.2677813212076823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,512,0.02717973391215007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,512,0.016990933815638223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,256,0.015178666512171427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,256,0.02105813423792521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,256,0.011965866883595784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,128,0.00900266667207082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,128,0.02030293345451355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,128,0.009382399916648864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,64,0.006052266558011373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,64,0.01863893270492554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,2048,0.11307733058929444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,65536,32,0.004496000210444133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,32,0.01804800033569336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,65536,512,0.029496532678604127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,3584,0.09567893346150716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,3072,0.07163306872049967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,1536,0.039287467797597245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,65536,1.2111701965332031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,65536,768,0.022638932863871256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,16384,0.6010517120361328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,51200,0.9614090601603189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,65536,2.3613438924153645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,16384,0.3679978688557943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,51200,1.8590741475423176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,12288,0.4552117347717285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,12288,0.27482239405314124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,10240,0.37503039042154945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,10240,0.20574399630228676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,10240,0.22979413668314613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,8192,0.30778239568074545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,8192,0.1683893362681071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,8192,0.14980160395304362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,7168,0.14590613047281903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,7168,0.30036373138427735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,6144,0.22553067207336425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,6144,0.12596053282419842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,6144,0.11419306596120198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,5120,0.1868874708811442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,5120,0.1127285321553548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,5120,0.11084907054901123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,4096,0.08959253629048666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,4096,0.1519818623860677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,3584,0.13422293663024903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,3584,0.07916266918182373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,3584,0.06941546599070231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,3072,0.11493866443634033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,3072,0.07016533215840658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,3072,0.06014933188756307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,2560,0.09721813201904297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,2560,0.0611413319905599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,2560,0.05173226594924927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,65536,1.1515071868896485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,2048,0.07691199779510498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,2048,0.05179093281428019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,1536,0.05852906703948975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,1536,0.0429535984992981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,1536,0.033505066235860186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,1024,0.04095040162404378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,1024,0.03358826637268066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,1024,0.024530132611592613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,768,0.031335467100143434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,768,0.028264532486597698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,768,0.019454934199651084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,512,0.02222826679547628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,51200,0.9011349360148111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,16384,0.29444586435953773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,512,0.014844800035158793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,512,0.023670399188995363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,256,0.0108106662829717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,128,0.0074432000517845156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,256,0.013452800114949546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,128,0.007737599809964498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,128,0.01596799989541372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,64,0.004066133250792822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,51200,32,0.004158933212359746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,64,0.01585706671079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,32,0.01591679950555166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,12288,0.2589973290761312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,7168,0.13236693541208905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,65536,0.37700907389322913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,65536,0.747650146484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,65536,0.3741482734680176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,16384,0.1856287956237793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,51200,0.2984842618306478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,4096,0.07825919787089029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,51200,0.5803125381469727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,16384,0.10992746353149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,16384,0.12394453684488933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,12288,0.14070506095886232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,12288,0.09743253389994302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,10240,0.0726645310719808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,10240,0.11648000081380208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,10240,0.06063786745071411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,8192,0.0945525328318278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,8192,0.06113280057907104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,7168,0.08268906275431315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,7168,0.0557365338007609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,6144,0.07218773365020752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,51200,2048,0.04770666758219401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,6144,0.05003840128580729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,51200,256,0.01936533252398173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,6144,0.04466773271560669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,5120,0.060090665022532144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,5120,0.044175998369852705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,5120,0.03822293281555176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,4096,0.04883840084075928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,4096,0.03827093442281087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,3584,0.04324800173441569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,4096,0.03166933258374532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,3584,0.03572266499201457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,3584,0.025225599606831867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,3072,0.04397866725921631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,3072,0.032708267370859787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,3072,0.02225173314412435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,2560,0.03184853394826253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,2560,0.029795199632644653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,2048,0.025909332434336345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,2048,0.016473600268363954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,2048,0.026837333043416338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,1536,0.020565332969029744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,1024,0.01453013320763906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,1536,0.013638400038083396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,1024,0.020321067174275717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,12288,0.0719210704167684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,768,0.011556266744931539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,768,0.009100799759229023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,768,0.01912320057551066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,512,0.00860053300857544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,512,0.007574399809042613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,512,0.017232000827789307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,256,0.005683200061321258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,256,0.015095466375350952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,128,0.003500800083080927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,128,0.014739200472831726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,128,0.0050240000089009605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,64,0.0030901332696278887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,64,0.014878933628400167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,16384,32,0.0031871999303499854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,32,0.0149536003669103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,16384,1536,0.023628799120585124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,51200,0.2892383893330892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,65536,0.5526496251424153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,65536,0.2928746541341146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,8192,0.04981013139088948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,7168,0.0441866676012675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,51200,0.43543148040771484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,16384,0.14440852801005047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,51200,0.23405332565307618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,16384,0.09944319725036621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,51200,0.2199882666269938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,12288,0.11789653301239014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,12288,0.06722666422526041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,12288,0.0664458672205607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,10240,0.04842986663182576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,10240,0.05787733395894369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,8192,0.07534933090209961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,8192,0.05049706697463989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,1024,0.010483200351397198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,8192,0.04018346468607585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,7168,0.0661952018737793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,7168,0.04515093167622884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,6144,0.0552127997080485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,6144,0.04083733161290486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,5120,0.045186134179433186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,5120,0.036950401465098065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,256,0.005378133555253347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,5120,0.027292799949645997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,4096,0.03671573400497437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,4096,0.03247359991073608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,3584,0.03298346598943074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,4096,0.022924800713857017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,3584,0.030292266607284547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,3584,0.020823466777801513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,3072,0.028867199023564653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,3072,0.027923200527826948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,3072,0.0186463991800944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,2560,0.02632533311843872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,10240,0.09219093322753906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,2560,0.025432533025741576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,2048,0.020035199324289956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,2048,0.0234442671140035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,16384,2560,0.019766400257746376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,1536,0.015723733107248943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,1536,0.0216213325659434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,1536,0.011755733688672384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,1024,0.011694932977358501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,1024,0.01950613260269165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,1024,0.009197866916656494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,768,0.009719467163085938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,768,0.01833066741625468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,768,0.00816319982210795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,512,0.007551999886830647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,512,0.015556266903877259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,65536,0.2773568153381348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,512,0.005625600119431814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,256,0.015438933173815408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,256,0.004049066702524821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,256,0.005241600175698599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,128,0.0035157332817713416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,128,0.014961066842079162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,128,0.004875733455022176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,64,0.003197866678237915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,12288,32,0.0032405334214369455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,64,0.014755200346310934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,12288,32,0.014877866705258688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,16384,0.07564373016357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,65536,0.2507370630900065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,65536,0.47524693806966145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,7168,0.03584106763203938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,6144,0.03148693243662516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,51200,0.1999029318491618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,51200,0.3723690668741862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,16384,0.1233728011449178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,16384,0.08245866298675537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,12288,0.09431040287017822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,12288,0.048595201969146726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,12288,0.06062933206558228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,10240,0.05268906752268473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,10240,0.07953920364379882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,8192,0.06646720170974732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,10240,0.04149866501490275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,8192,0.044276265303293864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,7168,0.053515732288360596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,8192,0.03445440133412679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,7168,0.04077226718266805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,7168,0.031014400720596313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,6144,0.04856319824854533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,6144,0.037102933724721274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,5120,0.04072426557540894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,2048,0.013558399677276612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,5120,0.033562668164571124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,4096,0.03188479940096538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,4096,0.02967573404312134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,4096,0.02058666745821635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,3584,0.027559467156728107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,16384,0.07429760297139486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,3072,0.024843732515970864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,3072,0.025718400875727337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,12288,2560,0.0173418660958608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,2560,0.021112533410390218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,2560,0.02398186723391215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,2560,0.014620799819628397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,2048,0.01726613243420919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,3584,0.02834986646970113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,2048,0.021976532538731892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,1536,0.013708800077438354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,2048,0.012645333011945089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,65536,0.23886613845825194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,1536,0.010843732953071594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,1024,0.011518933375676473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,1024,0.01844586730003357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,768,0.008451199531555176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,1024,0.008763733506202697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,768,0.007057066758473713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,768,0.01707306702931722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,512,0.006677333513895671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,512,0.015304533640543619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,256,0.0036501333117485045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,256,0.014863999684651694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,51200,0.18325866063435872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,128,0.003257599969704946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,128,0.014748799800872802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,128,0.004806399842103322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,64,0.0029333333174387617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,64,0.014869333306948344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,10240,32,0.002946133414904277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,32,0.014833066860834757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,6144,0.027265065908432008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,10240,1536,0.02038933237393697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,5120,0.023772799968719484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,65536,0.3736309369405111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,512,0.005909333129723867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,65536,0.199398406346639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,3584,0.0186901330947876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,51200,0.29704853693644206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,16384,0.09199893474578857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,51200,0.16118507385253905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,51200,0.14530347188313802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,16384,0.0507584015528361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,16384,0.06844159762064615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,12288,0.07318507035573324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,12288,0.049854934215545654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,12288,0.03983253240585327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,10240,0.05899519920349121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,10240,0.043620268503824874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,10240,0.03377066850662232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,8192,0.04779200156529744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,7168,0.04220159848531087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,8192,0.038634665807088214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,8192,0.028743465741475422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,3072,0.016546133160591125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,7168,0.03520533243815104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,6144,0.037069865067799884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,6144,0.02272746761639913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,6144,0.03265920082728068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,5120,0.031319467226664226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,4096,0.025523199637730913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,5120,0.019795199235280357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,4096,0.026306132475535076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,4096,0.01736533244450887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,3584,0.015735466281572977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,3584,0.02495786746342977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,3072,0.020205867290496827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,3072,0.023457066218058268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,3072,0.013703466455141703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,2560,0.01735573410987854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,2560,0.022127999862035116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,2560,0.012585600217183432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,2048,0.014288000265757241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,2048,0.02062079906463623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,2048,0.011015466849009196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,1536,0.011427199840545655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,1536,0.01913493275642395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,1536,0.00976746678352356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,1024,0.008644266923268636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,1024,0.01733760039011637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,768,0.007337599992752075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,768,0.015650133291880287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,768,0.00609493354956309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,512,0.005721599857012431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,512,0.015291733543078103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,10240,256,0.005284266670544943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,256,0.0035317334036032355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,256,0.014773333072662353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,5120,0.029270400603612263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,256,0.005109333495299021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,128,0.003160533308982849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,3584,0.02299626668294271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,64,0.0029845332105954488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,128,0.014587733149528503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,128,0.004863999783992767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,8192,32,0.002930133293072383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,64,0.014632532993952433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,8192,32,0.014521599809328715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,65536,0.3259317398071289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,65536,0.1627957344055176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,65536,0.18268693288167318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,51200,0.1291872024536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,51200,0.2574186642964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,51200,0.14737920761108397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,65536,0.18409172693888348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,16384,0.08950080076853434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,12288,0.06813759803771972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,16384,0.060585598150889076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,12288,0.04686719973882039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,12288,0.0359007994333903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,10240,0.054459734757741296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,10240,0.041414399941762284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,8192,0.04458986520767212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,8192,0.03576533397038777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,8192,0.02550293405850728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,7168,0.025590399901072185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,7168,0.02328959902127584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,6144,0.035410133997599284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,6144,0.030371199051539104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,6144,0.020693333943684895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,5120,0.02997013330459595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,5120,0.028203733762105304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,5120,0.017963733275731406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,4096,0.023139200607935586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,4096,0.025139200687408447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,4096,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,3584,0.02073813279469808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,3584,0.023834667603174844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,3072,0.018348799149195353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,3072,0.022576000293095908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,3072,0.012640000383059184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,2560,0.01575146714846293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,1024,0.007715199887752533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,2560,0.011693867047627766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,2560,0.021522132555643718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,2048,0.013266133268674216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,2048,0.01987839937210083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,2048,0.01025279959042867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,1536,0.010665599505106609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,1536,0.018677333990732826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,8192,512,0.005527466535568237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,1536,0.009066667159398396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,1024,0.008054399987061818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,10240,0.03071146607398987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,1024,0.006735999882221222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,1024,0.015500799814860026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,768,0.006854400038719177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,7168,0.03973013162612915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,7168,0.03315200010935466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,512,0.003960533440113068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,768,0.00595306654771169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,768,0.015356799960136414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,512,0.014988799889882406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,512,0.005515733361244201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,256,0.005251200000445048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,256,0.014827733238538107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,128,0.0031061333914597826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,64,0.0028362666567166646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,64,0.014533332983652749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,32,0.002882133424282074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,32,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,65536,0.2797727902730306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,65536,0.15601600011189778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,65536,0.1429866631825765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,51200,0.22136319478352867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,51200,0.12503999869028729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,16384,0.07489813168843587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,16384,0.04521493514378865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,16384,0.05146026611328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,7168,256,0.0033642667035261786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,12288,0.057384534676869714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,12288,0.041966934998830155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,7168,128,0.014801067113876343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,12288,0.032014934221903484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,10240,0.04808106819788615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,10240,0.038045867284138994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,8192,0.03860053221384684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,8192,0.03393599987030029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,8192,0.022898133595784506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,7168,0.03448319832483927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,7168,0.03075946569442749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,6144,0.029445334275563555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,6144,0.028620799382527668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,6144,0.0186954657236735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,3584,0.014446933070818582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,5120,0.025049599011739095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,5120,0.026612265904744463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,5120,0.016233600179354348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,4096,0.0213919997215271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,4096,0.024229333798090617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,4096,0.0139082670211792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,3584,0.018678400913874307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,3584,0.023061333100001018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,3584,0.013091199596722922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,3072,0.016965333620707193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,3072,0.021596799294153847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,3072,0.011688533425331115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,2560,0.014261333147684732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,2560,0.02053226629892985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,2560,0.010889599720637005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,2048,0.0123690664768219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,2048,0.019333332777023315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,2048,0.009549867113431294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,1536,0.008686932921409606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,1536,0.018221867084503175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,1024,0.007525333265463512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,1024,0.015019733707110086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,768,0.006392533580462138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,768,0.015146666765213012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,768,0.005874133110046387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,512,0.003790933390458425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,512,0.014932266871134438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,7168,128,0.004874666531880697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,256,0.0033461332321166994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,256,0.014905599753061929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,256,0.0050911997755368555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,128,0.003017599880695343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,128,0.014692266782124838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,1536,0.010046933094660442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,64,0.002881066749493281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,64,0.014571733276049294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,6144,32,0.002885333448648453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,6144,32,0.014616533120473226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,51200,0.11332586606343586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,16384,0.04017920096715291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,7168,0.020964266856511433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,10240,0.027331199248631793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,65536,0.13671679496765138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,65536,0.23793387413024902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,65536,0.12321386337280274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,16384,0.06905492941538492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,51200,0.10763626893361408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,51200,0.18811839421590168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,51200,0.09852266311645508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,16384,0.04615360101064046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,12288,0.04856853485107422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,16384,0.03563946485519409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,12288,0.04083520174026489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,10240,0.03378880023956299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,10240,0.024602667490641276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,8192,0.030362667640050252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,8192,0.02042133410771688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,7168,0.028065067529678345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,7168,0.027846399943033857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,7168,0.01876586675643921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,6144,0.024654932816823325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,6144,0.025993599494298296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,6144,0.01660160024960836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,5120,0.021234132846196494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,5120,0.024156800905863442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,1024,0.006495999793211619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,4096,0.017450666427612303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,4096,0.022403200467427574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,4096,0.01269546647866567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,3584,0.015782399972279867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,3584,0.021458133061726888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,3584,0.012081066767374676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,3072,0.014044800400733947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,10240,0.04352960189183553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,3072,0.02044266661008199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,512,0.005340800185998281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,2560,0.012211199601491292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,3072,0.010956799983978272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,8192,0.03371200164159139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,2560,0.0195743997891744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,2048,0.010364799698193868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,2560,0.01032533347606659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,1536,0.008674133817354839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,2048,0.008884267012278239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,2048,0.018631466229756675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,1536,0.016908800601959227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,1024,0.006822399795055389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,1536,0.00740479975938797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,6144,128,0.004702933132648468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,768,0.005489066739877065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,1024,0.015242666999499003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,1024,0.006309333443641663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,768,0.01514346698919932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,512,0.003647999962170919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,768,0.0057888001203536986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,512,0.005432533224423727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,256,0.0032127998769283296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,512,0.01498240033785502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,256,0.005058133105436961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,256,0.014598400394121806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,128,0.004691199958324432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,64,0.00279573326309522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,128,0.014575999975204468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,32,0.0027978666126728057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,64,0.01441493332386017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,5120,32,0.01446613371372223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,65536,0.10618666807810466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,65536,0.18816320101420086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,51200,0.14754667282104492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,16384,0.049848532676696776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,51200,0.08844693501790366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,51200,0.0842357317606608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,16384,0.03086613416671753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,16384,0.038957866032918294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,12288,0.03882346550623576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,12288,0.024910932779312132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,10240,0.03303359945615132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,10240,0.030472532908121748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,10240,0.021590399742126464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,8192,0.0272597332795461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,8192,0.027131734291712443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,8192,0.018041600783665977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,7168,0.024616533517837526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,7168,0.025316266218821208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,7168,0.016339199741681416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,6144,0.021987199783325195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,6144,0.023746132850646973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,6144,0.014842666188875833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,5120,0.019015467166900633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,12288,0.028919466336568195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,5120,0.013092266519864401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,5120,0.022363734245300294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,4096,0.015904000401496886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,3584,0.01318933367729187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,4096,0.02045546571413676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,3584,0.02007253368695577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,5120,128,0.0029663999875386557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,3072,0.011731200416882833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,3584,0.011174399654070537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,2560,0.010276266932487487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,3072,0.01014400025208791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,2560,0.01874026656150818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,2560,0.009542399644851684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,2048,0.008920533458391826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,1536,0.007635200023651123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,2048,0.0176746666431427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,1536,0.015434666474660238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,1024,0.0061034664511680605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,65536,0.10877119700113932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,1024,0.015154133240381876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,768,0.004121600091457367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,12288,0.033370665709177655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,768,0.015072000026702882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,5120,5120,0.014696533481280008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,768,0.005701333284378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,512,0.0035018667578697203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,512,0.005241600175698599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,512,0.014808533589045205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,256,0.0031541332602500914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,256,0.004851200183232625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,256,0.014774399995803832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,128,0.0029472000896930695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,128,0.014619732896486918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,128,0.004749866823355356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,64,0.0027562665442625684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,64,0.014509866635004679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,4096,32,0.0028149334092934927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,32,0.014512000481287637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,4096,0.011534933249155681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,4096,3072,0.019409066438674925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,65536,0.16806507110595703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,65536,0.09951786994934082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,2048,0.008090666433175405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,65536,0.10252587000528972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,51200,0.08270826339721679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,16384,0.04709546566009522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,51200,0.13045866489410402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,16384,0.036161065101623535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,12288,0.03566293319066365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,16384,0.030049065748850506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,10240,0.030503465731938677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,12288,0.031140265862147014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,10240,0.028842665751775104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,8192,0.024528000752131143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,8192,0.017603200674057008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,8192,0.025043199459711712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,7168,0.02226240038871765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,7168,0.024065067370732628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,7168,0.01629866659641266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,6144,0.019691733519236247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,6144,0.02284053365389506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,6144,0.014678399761517844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,5120,0.017268266280492148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,5120,0.013159466783205667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,4096,0.013202133774757385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,4096,0.02026453415552775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,4096,0.011592533191045125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,3584,0.011974400281906128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,3584,0.019502933820088705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,3584,0.01090773344039917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,3072,0.010853333274523418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,3072,0.019248000780741372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,12288,0.023836799462636313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,2560,0.009574400385220845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,2560,0.018780799706776936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,2560,0.009325866897900898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,1536,0.006997333467006683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,2048,0.008329600095748901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,2048,0.01588266690572103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,2048,0.007701333363850911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,4096,1024,0.005996799965699514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,1536,0.0071829333901405334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,1024,0.0046069333950678505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,1536,0.01576959987481435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,5120,0.021746132771174112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,1024,0.015258666872978211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,768,0.003914666672547659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,768,0.014964266618092855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,768,0.005610666672388713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,512,0.0034954667091369627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,512,0.005264000097910563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,256,0.00308693324526151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,256,0.01467626690864563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,256,0.005026133358478546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,128,0.0028938665986061097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,128,0.014712533354759217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,64,0.002665599932273229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,128,0.004752000172932943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3584,32,0.0027082666754722597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,64,0.01455573340257009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,32,0.014513066411018372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,65536,0.14304107030232746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,65536,0.08769386609395345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,65536,0.09433920383453369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,51200,0.08166613578796386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,51200,0.11343680222829182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,51200,0.0710208018620809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,16384,0.03974613348642985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,16384,0.033514666557312014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,10240,0.020138667027155558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,16384,0.02784000039100647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,12288,0.03147733410199483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,1024,0.006168533364931742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,12288,0.02853226661682129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,12288,0.02247999906539917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,10240,0.02644266684850057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,8192,0.02190613349278768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,10240,0.026902399460474652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,8192,0.024705066283543905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,8192,0.016389333208402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,7168,0.01989439924558004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3584,512,0.014908799529075622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,7168,0.023562665780385336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,6144,0.01739733417828878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,6144,0.022114133834838866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,6144,0.013940266768137612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,5120,0.015320533514022827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,5120,0.020860799153645835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,5120,0.012494933605194092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,4096,0.013448533415794373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,4096,0.019718400637308755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,4096,0.01111893355846405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,3584,0.01258026659488678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,3584,0.010548266768455505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,3584,0.01924053430557251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,3072,0.01120746632417043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,3072,0.018588799238204955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,3072,0.009634133179982502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,2560,0.008985599875450135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,2560,0.017090133825937905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,2560,0.0086325337489446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,2048,0.007659733295440674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,3072,0.009969066580136616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,2048,0.007490133245786031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,2048,0.015811199943224587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,1536,0.006760533154010773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,1024,0.004439466694990794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,1536,0.006671999891599019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,1536,0.015553067127863566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,768,0.003864533454179764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,1024,0.0059125334024429325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,1024,0.015385599931081137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,512,0.0035445332527160645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,768,0.005510400235652924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,768,0.015158399939537048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,256,0.003172266731659571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,512,0.005302399893601735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,512,0.014962133765220643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,51200,0.07509120305379233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,128,0.002940800040960312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,256,0.014627200365066529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,64,0.0027776000400384264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,128,0.014694399634997048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,3072,32,0.0027061333258946735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,64,0.014673067132631936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,3072,32,0.014509866635004679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,65536,0.11996586322784424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,65536,0.07581973075866699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3584,1536,0.006877866884072621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,51200,0.06098026831944784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,51200,0.07354880174001058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,16384,0.03314453363418579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,16384,0.030404265721638995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,10240,0.018683733542760213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,12288,0.026162133614222212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,7168,0.015446399648984274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,12288,0.026362667481104534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,10240,0.022789333264033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,10240,0.024702932437260947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,10240,0.018994132677714028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,8192,0.019074134031931558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,8192,0.022792534033457438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,7168,0.017359999815622966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,7168,0.02190933426221212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,256,0.005020800232887268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,6144,0.01534293293952942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,6144,0.020793600877126058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,3072,128,0.004844800134499868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,6144,0.013716266552607218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,5120,0.0135754664738973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,5120,0.012392533818880717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,5120,0.020002132654190062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,4096,0.011662933230400085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,4096,0.019093332688013713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,4096,0.011265066266059876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,65536,0.09275200366973876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,3584,0.010903466741243999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,3584,0.01869759956995646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,3584,0.010406399766604107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,3072,0.00997440020243327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,2560,0.008292266726493835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,3072,0.009248000383377076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,3072,0.017774933576583864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,2048,0.007233066856861115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,2560,0.016006400187810264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,2048,0.00729066679875056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,2048,0.016051200032234193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,1536,0.005876266459623972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,1536,0.01577600042025248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,1536,0.006580266853173573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,1024,0.004134399940570196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,1024,0.01511360009511312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,1024,0.005930666625499725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,16384,0.027513599395751952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,768,0.00384853333234787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,512,0.003499733408292135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,768,0.015097600221633912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,512,0.005272533496220907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,512,0.014660267035166421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,256,0.0030847998956839246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,256,0.014541866381963095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,51200,0.09551786581675212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,128,0.002812800059715907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,128,0.014620799819628397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,128,0.004741333425045013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,64,0.0026496000587940215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,64,0.014551466703414917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2560,32,0.0026933332284291584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2560,32,0.014544000228246054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,12288,0.02194026708602905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,65536,0.0964693307876587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,65536,0.061679999033610024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,8192,0.015340800086657206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,51200,0.0751039981842041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,51200,0.05197759866714478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,16384,0.02729066610336304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,51200,0.06763412952423095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,7168,0.015153066317240397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,256,0.004947199920813243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,16384,0.026811732848485308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,12288,0.021755733092625937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,12288,0.024149332443873087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,10240,0.019052799542744955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,10240,0.023180800676345825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,10240,0.017811199029286705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,8192,0.015871999661127727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,8192,0.02117439905802409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,8192,0.01539520025253296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,7168,0.014358400305112203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,7168,0.020795732736587524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,6144,0.01290773351987203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,6144,0.020137600104014077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,2560,0.00827519992987315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,5120,0.0116565336783727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,5120,0.019667200247446694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,5120,0.011634133259455363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,16384,0.02524799903233846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,4096,0.010188800096511842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,4096,0.01833066741625468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2560,768,0.00547733356555303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,3584,0.009603200356165568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,3584,0.009637332955996196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,3584,0.017824000120162962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,3072,0.008963200449943542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,3072,0.016266666849454246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,3072,0.008684800068537394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,2560,0.007639466722806294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,2560,0.015884799758593242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,2048,0.006750933329264323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,2048,0.015913599729537965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,1536,0.0048096001148223875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,1536,0.0155157337586085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,1536,0.006492800017197926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,1024,0.003977599988381068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,1024,0.015165866414705912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,1024,0.005701333284378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,65536,0.08522346814473471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,768,0.0037429332733154297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,768,0.005443199972311656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,768,0.01495039959748586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,512,0.0033439998825391137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,512,0.01487573285897573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,256,0.003033600002527237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,256,0.014651733636856078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,12288,0.019977599382400513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,128,0.002776533365249634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,128,0.01446613371372223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,128,0.004635733366012573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,64,0.0026410666604836782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,64,0.014461867014567056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,2048,32,0.0026378666361172995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,2048,32,0.014446933070818582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,6144,0.012905599673589072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,65536,0.07435093720753988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,65536,0.05142293373743693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,51200,0.05921386480331421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,65536,0.08386133511861166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,51200,0.04352426528930664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,16384,0.021919999519983926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,51200,0.06649920145670572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,16384,0.024260266621907552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,16384,0.024755199750264488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,12288,0.01771946748097738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,12288,0.022101332743962608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,12288,0.019913599888483683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,10240,0.015426133076349893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,10240,0.02108586629231771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,10240,0.017262933651606242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,8192,0.0132533331712087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,8192,0.01990613341331482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,8192,0.015037866433461508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,7168,0.012108799815177918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,7168,0.01938026746114095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,7168,0.013656533757845559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,4096,0.010134399930636088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,6144,0.01118293305238088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,6144,0.012444800138473511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,6144,0.01918720006942749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,5120,0.011282133062680562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,5120,0.018873600165049235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,4096,0.008958933750788371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,4096,0.016938666502634682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,7168,0.014283733566602072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,2560,0.007905066510041555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,3584,0.008566400408744812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,3584,0.01695573329925537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,3584,0.008937600255012512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,3072,0.007964799801508587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,3072,0.016659200191497803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,2048,0.0070816000302632645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,2560,0.006855466465155284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,2560,0.01630826691786448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,2560,0.007614933451016744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,2048,0.005381333331267038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,2048,0.01585706671079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,2048,0.006856533388296763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,1536,0.004667733112970988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,1536,0.015461333592732749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,1536,0.006498133142789205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,1024,0.004088533421357473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,1024,0.015108266472816467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,1024,0.005684266487757364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,768,0.0037205333511034647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,768,0.015090133746465048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,512,0.0033642667035261786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,512,0.014820266763369241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,512,0.005163733164469401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,256,0.0030847998956839246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,5120,0.010101333260536194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,256,0.014411733547846476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,256,0.004814933240413666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,128,0.0028778667251269023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,128,0.004757333298524221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,64,0.0026634665826956432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,64,0.014556800325711569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1536,32,0.0027200000981489818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,32,0.014575999975204468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,65536,0.04952853520711263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,65536,0.039738667011260984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,65536,0.08303786913553873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,51200,0.039979732036590575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,51200,0.03461333513259888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,768,0.005403733253479004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,16384,0.015995732943216958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,16384,0.02141653299331665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,16384,0.024652800957361855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,12288,0.013269333044687906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,12288,0.019723733266194664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,2048,256,0.004852266609668731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,12288,0.019771732886632285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,10240,0.011960533261299134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1536,128,0.0144405335187912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,10240,0.019418666760126747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,10240,0.01726933320363363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,8192,0.010450133681297302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,7168,0.00974826713403066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,8192,0.01495253344376882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,8192,0.018506666024525963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,7168,0.013412266969680786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,7168,0.017617066701253258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,6144,0.008661333719889324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,5120,0.008322133123874665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,6144,0.011774933338165284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,6144,0.017787732680638633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,5120,0.010403199990590414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,4096,0.007525333265463512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,4096,0.009701333443323771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,5120,0.017912532885869345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,3584,0.006975999971230824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,3072,0.006500266492366791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,3584,0.017053866386413576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,3072,0.008019199967384339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,3072,0.016471466422080992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,2560,0.005964800218741099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,2560,0.007565866907437642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,2560,0.016090666254361473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,3072,0.007995733122030894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,2048,0.015801599621772765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,2048,0.006779733300209046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,1536,0.004638933142026265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,1536,0.01550933321317037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,1024,0.003973333289225897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,1024,0.015129599968592325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,1024,0.0056639999151229855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,768,0.003634133438269297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,768,0.015005866686503092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,768,0.005284266670544943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,512,0.0033802665770053864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1536,512,0.005032533407211303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,512,0.014651733636856078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,512,0.005131733417510986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,256,0.003035733352104823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,256,0.004851200183232625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,256,0.014600533246994018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,128,0.00275093341867129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,128,0.004614399870236715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,128,0.01488746702671051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,64,0.003033600002527237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,32,0.0030069333811601003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,64,0.014937600493431092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,32,0.014647466937700906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,4096,0.00929813285668691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,1024,4096,0.01715839902559916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,65536,0.038957866032918294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,65536,0.03431573311487834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,51200,0.031838933626810714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,51200,0.03023359974225362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,65536,0.083351469039917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,1024,2048,0.0053845331072807315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,51200,0.06626559893290201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,12288,0.011380267143249512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,16384,0.0244159996509552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,12288,0.019495467344919838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,12288,0.018969599405924478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,10240,0.01032426655292511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,10240,0.018632533152898152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,10240,0.016884267330169678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,8192,0.009320533275604248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,8192,0.017153066396713258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,8192,0.014367999633153281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,7168,0.008529067039489746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,7168,0.017908267180124917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,51200,0.06602133512496948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,6144,0.007826133569081625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,6144,0.017475199699401856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,5120,0.007315200070540111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,5120,0.01806400020917257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,5120,0.010391466816266378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,4096,0.006233599781990051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,4096,0.017233065764109292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,4096,0.00932373305161794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,3584,0.0064629331231117245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,3584,0.01691626707712809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,3584,0.008861866593360902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,3072,0.006101333101590474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,3072,0.016556800405184428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,3072,0.008021333316961924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,2560,0.006093866626421611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,16384,0.01338879962762197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,2560,0.00768746683994929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,16384,0.020588799317677816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,2048,0.0053269331653912864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,3584,0.008950400352478027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,2048,0.006735999882221222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,2048,0.015846400459607442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,1536,0.004668800036112467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,1024,0.003973333289225897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,1536,0.00643093337615331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,1536,0.015562666455904641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,1024,0.00581226646900177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,768,0.0036042665441830954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,1024,0.015228799978892007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,512,0.0033919999996821085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,768,0.015228799978892007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,7168,0.01304639975229899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,512,0.0051701332132021586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,512,0.014994133512179056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,256,0.0032149332265059153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,1024,1536,0.0064416001240412395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,128,0.0029311999678611755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,256,0.014703999956448874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,64,0.00278613343834877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,128,0.0046741331617037455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,128,0.01458026667435964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,768,32,0.002844800055027008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,64,0.014532267053922018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,32,0.01458133359750112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,65536,0.027401600281397504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,51200,0.022652800877889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,65536,0.02805440028508504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,65536,0.08327253659566244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,51200,0.025198932488759356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,16384,0.010472533106803895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,51200,0.06601813236872354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,16384,0.01912426749865214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,16384,0.024341332912445068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,12288,0.009097599983215332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,10240,0.008516266942024231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,12288,0.019232000907262167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,12288,0.01720959941546122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,8192,0.007693866888682048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,10240,0.01684266726175944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,10240,0.017525333166122436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,7168,0.006773333251476288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,8192,0.017362133661905924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,7168,0.012838400403658547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,7168,0.01776533325513204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,6144,0.00617386649052302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,6144,0.017490132649739584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,6144,0.011539199948310852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,5120,0.0065311998128890995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,5120,0.018490666151046754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,768,2560,0.016193067034085594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,4096,0.005907199780146281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,4096,0.009226666887601216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,4096,0.017361066738764443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,3584,0.006291200220584869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,3584,0.008880000313123066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,3584,0.017174400885899863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,3072,0.0060032000144322716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,3072,0.007997866471608479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,2560,0.006071466704209646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,3072,0.016565333803494772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,2560,0.01660480002562205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,2048,0.0053845331072807315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,256,0.0049002667268117275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,2048,0.015894400080045064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,1536,0.004726399978001913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,2048,0.006738133231798808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,1536,0.006354133288065593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,1536,0.015629866719245912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,1024,0.004165333261092504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,1024,0.01514346698919932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,6144,0.011714133620262145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,768,0.0037461332976818085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,768,0.005354666709899902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,768,0.015033599734306336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,512,0.00346666673819224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,512,0.004923733572165171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,512,0.014753066500027976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,256,0.0031061333914597826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,256,0.01456000010172526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,128,0.002883200099070867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,128,0.01442026694615682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,128,0.004661333560943603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,64,0.0026805333793163298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,64,0.014520532886187234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,512,32,0.0026805333793163298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,512,32,0.01458346645037333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,65536,0.01704746683438619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,65536,0.022836265961329143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,65536,0.08305066426595052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,51200,0.014502400159835815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,51200,0.021186133225758873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,51200,0.06574613253275553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,16384,0.007526400188604991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,768,768,0.005392000079154968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,16384,0.023733333746592204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,16384,0.01776960094769796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,12288,0.006259199976921081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,12288,0.017773866653442383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,12288,0.018781866629918417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,10240,0.006296533346176148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,10240,0.017797333002090455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,10240,0.016450132926305136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,256,0.004845866560935974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,8192,0.017063466707865398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,7168,0.006076799829800924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,7168,0.017666133244832356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,7168,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,6144,0.005876266459623972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,6144,0.017296000321706136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,6144,0.011590400338172912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,5120,0.0062720000743865965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,5120,0.017941333850224814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,5120,0.010420266787211101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,5120,0.010487467050552368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,4096,0.005854933460553487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,4096,0.017169066270192466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,4096,0.009256533781687419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,3584,0.006313600142796834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,3584,0.016925867398579916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,3584,0.008842666943868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,3072,0.0060245335102081295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,3072,0.016427733500798545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,3072,0.008070399860541026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,2560,0.0060703997810681665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,2560,0.007628799974918365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,2560,0.016219733158747356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,2048,0.005286400020122528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,2048,0.01585599978764852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,1536,0.004749866823355356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,1536,0.01548693378766378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,1536,0.006483200192451477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,1024,0.004089600096146265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,8192,0.0060586666067441305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,1024,0.005689600110054016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,1024,0.005659733215967814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,1024,0.015202132860819497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,512,0.0033173332611719764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,768,0.005251200000445048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,768,0.015029333035151162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,256,0.0030591999491055804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,512,0.014674133062362671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,256,0.004739200075467428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,256,0.014596266547838846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,128,0.0028597332537174227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,128,0.014467199643452963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,128,0.0047189335028330484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,64,0.0027200000981489818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,32,0.0027221334477265675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,512,8192,0.01416106621424357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,65536,0.011543466647466024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,32,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,65536,0.019357866048812865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,51200,0.010202667117118836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,51200,0.01830186645189921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,51200,0.06542400121688843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,16384,0.005842133363087972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,16384,0.01771946748097738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,2560,0.007477333148320516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,256,768,0.003732266773780187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,12288,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,12288,0.01731520096460978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,8192,0.0139957328637441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,256,64,0.014383999506632486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,10240,0.0059562668204307554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,10240,0.016394666830698647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,10240,0.0173909326394399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,8192,0.005826133489608765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,8192,0.017115734020868936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,8192,0.014045866330464682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,7168,0.0058442667126655575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,7168,0.01793173352877299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,7168,0.012822399536768595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,6144,0.005619200070699056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,6144,0.017199999094009398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,5120,0.006002133091290792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,5120,0.017427200078964235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,5120,0.010268800457318624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,4096,0.005650133391221364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,4096,0.01680319905281067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,4096,0.009218133489290873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,3584,0.006053333481152853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,3584,0.016743467251459757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,2048,0.006659199794133504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,3584,0.008718933661778767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,3072,0.0056991999348004665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,3072,0.016839466492335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,2560,0.005705599983533224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,2560,0.015759999553362526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,2560,0.007549866537253062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,2048,0.0050911997755368555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,2048,0.015612799922625223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,2048,0.006681600213050842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,256,512,0.005076266825199127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,1536,0.004564266900221507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,1536,0.015581867098808289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,1536,0.006401066482067108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,1024,0.003874133278926214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,1024,0.015006933609644571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,1024,0.005598933498064677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,768,0.00373333344856898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,768,0.015024000406265258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,512,0.003369600077470144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,512,0.014957867066065469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,65536,0.08228800296783448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,256,0.003018666555484136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,256,0.014509866635004679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,256,0.004829866687456766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,128,0.0028319999575614927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,128,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,16384,0.023613866170247397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,128,0.004590933521588644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,64,0.0026378666361172995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,128,32,0.0027114666998386385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,64,0.014591999848683677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,128,32,0.014447999993960061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,51200,0.007162666817506154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,65536,0.018630399306615194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,16384,0.005748266478379568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,51200,0.01856106718381246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,12288,0.005919999877611796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,16384,0.01757226586341858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,10240,0.005922133227189382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,12288,0.01716266671816508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,8192,0.005850666761398315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,10240,0.01750719944636027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,7168,0.005691733459631602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,8192,0.017053866386413576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,6144,0.005677866439024607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,7168,0.017505067586898803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,5120,0.005894400179386139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,6144,0.01715839902559916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,5120,0.01768640081087748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,4096,0.016962132851282754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,3584,0.00595306654771169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,3584,0.0165994664033254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,3072,0.005659733215967814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,3072,0.01624853312969208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,2560,0.005599999924500784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,6144,0.011500799655914306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,2048,0.005075199902057648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,2560,0.01599679986635844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,1536,0.004420266548792521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,2048,0.015660799543062844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,1024,0.0038549333810806276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,1536,0.015367466211318969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,768,0.0035135999321937563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,1024,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,512,0.0032000000278155005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,768,0.014740266402562461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,256,0.002994133283694585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,65536,0.00853546659151713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,512,0.01479039986928304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,128,0.002842666705449422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,256,0.014667733510335287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,64,0.00266239990790685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,128,0.014473600188891092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,32,0.0026549334327379864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,64,0.014481066664059957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,65536,0.008177066842714947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,3072,0.007912533481915791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,65536,0.018977065881093345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,51200,0.007632000247637431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,16384,0.005804799993832906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,51200,0.01842666665712992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,16384,0.017273600896199545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,12288,0.005764266848564148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,12288,0.01883093317349752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,10240,0.005884799857934316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,12288,0.017043199141820273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,8192,0.005815466741720835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,10240,0.017602133750915527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,7168,0.00565119981765747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,8192,0.016902399063110352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,6144,0.005509333312511444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,7168,0.017399466037750243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,5120,0.005942399799823761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,6144,0.0168938676516215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,4096,0.0054837331175804135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,5120,0.01741546591122945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,3584,0.005983999868233999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,4096,0.016687999169031777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,3072,0.0056320001681645715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,3584,0.01651093363761902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,2560,0.00561706672112147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,3072,0.015957333644231162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,2048,0.005050666630268097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,2560,0.016037333011627197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,1536,0.004678399860858917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,2048,0.01546239952246348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,1024,0.003849600007136663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,1536,0.015317333738009134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,768,0.0035189333061377203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,1024,0.015160533785820007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,512,0.0031989333530267083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,256,0.0029077333708604175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,512,0.01436906655629476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,128,0.0027104000250498454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,256,0.014459733168284097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,128,0.0141184002161026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,64,0.002630399912595749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,512,0.005037866532802582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,32,32,0.0026954665780067446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,64,0.014141866564750671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,32,0.014350933829943338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,64,32,0.01432213286558787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,5,64,4096,0.005725866556167603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,5,128,768,0.005308799942334493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,16384,0.7480480194091796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,16384,0.39390827814737955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,16384,0.3551679929097494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,12288,0.29822187423706054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,12288,0.5646783828735351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,51200,1.179100799560547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,51200,1.0944756825764974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,12288,0.2705696105957031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,5,32,768,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,10240,0.26839892069498694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,8192,0.20518506368001305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,8192,0.18059306144714354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,8192,0.433132807413737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,10240,0.4752597490946452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,7168,0.3290165265401205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,7168,0.17917866706848146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,51200,2.3363466898600262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,7168,0.15904213587443033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,6144,0.1558890660603841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,6144,0.15512320200602214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,6144,0.2844757397969564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,5120,0.13290133476257324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,5120,0.23645119667053222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,4096,0.1934613386789958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,4096,0.09347733656565348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,5120,0.13465813000996907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,3584,0.16729386647542316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,4096,0.10868373711903889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,3584,0.0980949322382609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,3072,0.086189866065979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,3072,0.1443178653717041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,2560,0.12061653137207032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,2560,0.07447466850280762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,3072,0.08285653591156006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,2048,0.09687999884287515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,2048,0.06451093355814616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,2048,0.05495040019353231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,1536,0.05744426647822062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,1024,0.05047253370285034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,1536,0.03965760072072347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,1024,0.03925973176956177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,1024,0.03153066635131836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,768,0.03349013328552246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,768,0.022638932863871256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,512,0.031104000409444173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,512,0.02682773272196452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,256,0.017164800564448038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,256,0.02065066695213318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,256,0.011915733416875202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,128,0.02023680011431376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,128,0.009241599837938945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,3584,0.09104106426239014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,64,0.00658240020275116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,64,0.01882666746775309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,32,0.0044608001907666525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,32,0.018901334206263224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,1536,0.07429546515146891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,768,0.03929493427276611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,128,0.009020800391832989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,10240,0.26550079981486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,2560,0.06106346845626831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,512,0.017042134205500284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,65536,1.208348846435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,51200,0.9491039911905924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,65536,1.143927510579427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,51200,1.8551167805989581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,16384,0.3567925453186035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,16384,0.5999946594238281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,16384,0.3297077178955078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,65536,2.3568351745605467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,12288,0.4530303955078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,12288,0.23598079681396483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,12288,0.2607903957366943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,10240,0.3749589284261068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,10240,0.20405759811401367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,10240,0.1886965274810791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,8192,0.1659968058268229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,7168,0.13282346725463867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,7168,0.14721919695536295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,8192,0.3558453241984049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,6144,0.22577919960021972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,7168,0.2951786677042643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,6144,0.1259872039159139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,6144,0.11260586579640705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,5120,0.18698986371358234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,4096,0.15360320409138997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,4096,0.0887391964594523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,5120,0.12265066305796306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,4096,0.07852479616800943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,3584,0.13200426896413167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,3584,0.07982613245646158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,3584,0.06860160032908122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,3072,0.07018346786499023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,3072,0.060141865412394205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,3072,0.11385280291239422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,2560,0.09363626639048259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,2048,0.07764586607615152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,2560,0.05133866469065348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,2560,0.06110613346099854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,2048,0.0425439993540446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,1536,0.05896639823913574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,2048,0.05767253239949545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,1024,0.04086933135986328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,1536,0.04267306725184123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,1536,0.03773013353347778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,768,0.03139946659406026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,1024,0.02439039945602417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,512,0.022046933571497597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,768,0.028616533676783247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,512,0.023571199178695677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,256,0.011745066444079081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,512,0.014860799908638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,128,0.007500799993673961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,256,0.019206400712331137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,128,0.015083733201026916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,64,0.004048000027736028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,64,0.015478400389353433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,51200,32,0.0040522667268912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,32,0.01548906664053599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,51200,0.897329076131185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,51200,1024,0.036124801635742186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,768,0.019346133867899577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,65536,0.7483221054077148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,8192,0.14842880566914876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,65536,0.3776597340901693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,51200,0.30076373418172203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,65536,0.35420052210489905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,51200,0.5807050704956055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,5120,0.10516800085703533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,16384,0.21936532656351723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,16384,0.12332800229390461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,12288,0.08428160349527994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,12288,0.1663797378540039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,10240,0.07235093116760254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,10240,0.11700906753540039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,51200,0.2848885218302409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,10240,0.0629792014757792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,8192,0.09425280094146729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,8192,0.06087573369344076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,7168,0.08269440333048503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,7168,0.05535893440246582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,6144,0.07190186977386474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,6144,0.04961920181910197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,6144,0.038726401329040525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,5120,0.06002666552861532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,256,0.010781866312026978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,5120,0.03341013193130493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,51200,128,0.00793280005455017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,4096,0.048913065592447916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,4096,0.03789546489715576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,3584,0.043253334363301595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,3584,0.035341866811116535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,16384,0.10612800121307372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,3072,0.044078934192657473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,3072,0.03262186646461487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,3072,0.02229866584142049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,2560,0.031880533695220946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,2560,0.029688533147176104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,2560,0.019824000199635823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,2048,0.025869866212209065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,2048,0.026608000199000042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,2048,0.01634773313999176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,1536,0.02044373353322347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,5120,0.04411520163218181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,1536,0.023516800006230673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,1024,0.014475733041763306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,1024,0.020541866620381675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,1024,0.01051626702149709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,768,0.011502933502197266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,768,0.019102933009465535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,768,0.009060266613960265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,512,0.008657067020734151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,512,0.01755519906679789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,512,0.0076341331005096436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,256,0.005718400080998739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,256,0.015196800231933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,256,0.005454933146635691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,128,0.0034282666941483817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,128,0.014865066607793173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,12288,0.08410453001658122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,64,0.003115733216206233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,64,0.014899200201034546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,32,0.0031850665807724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,32,0.014855466286341348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,8192,0.05783893267313639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,7168,0.04426026741663615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,4096,0.02758079965909322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,65536,0.2924031893412272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,65536,0.27649707794189454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,65536,0.553328005472819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,3584,0.02510826587677002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,16384,0.07658560276031494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,16384,0.14522453943888347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,16384,0.09928106466929118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,51200,0.4341898600260417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,51200,0.23377386728922525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,12288,0.11019093195597332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,12288,0.06740907033284506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,12288,0.05782080094019572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,10240,0.10779626369476318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,10240,0.05840426683425903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,8192,0.07558720111846924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,8192,0.05044693152109782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,7168,0.06630613406499228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,7168,0.04608533382415771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,6144,0.05481813351313273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,7168,0.035979731877644854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,6144,0.04129386742909749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,5120,0.045499734083811444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,5120,0.036559998989105225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,4096,0.03967146476109822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,4096,0.03269333243370056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,1536,0.013482667009035745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,4096,0.02256106734275818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,3584,0.03283093372980754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,3584,0.030572799841562907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,3584,0.02100373307863871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,3072,0.02868799964586894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,3072,0.02790293296178182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,2560,0.02477226654688517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,2560,0.025689599911371867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,2560,0.01642346680164337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,2048,0.0205567995707194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,2048,0.0234389324982961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,2048,0.013732266426086426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,51200,0.22745386759440103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,1536,0.016215466459592185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,1536,0.011566932996114094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,1536,0.022461867332458495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,1024,0.011735467116038005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,1024,0.0093642661968867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,1024,0.019242666165033975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,128,0.0048426667849222815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,768,0.008224000036716462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,768,0.01816320021947225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,512,0.007681066791216533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,512,0.015309866269429526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,512,0.005846400062243144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,256,0.00396373321612676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,256,0.015227733055750528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,5120,0.027246934175491334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,256,0.005377066632111868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,128,0.0035690667728583017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,64,0.0032511999209721885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,128,0.014947199821472168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,32,0.003143466760714849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,64,0.01497066617012024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,32,0.014914133151372275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,65536,0.24981013933817545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,65536,0.4743786811828613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,768,0.009859200318654377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,65536,0.24251200358072916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,10240,0.04919039805730184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,51200,0.3716224034627279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,8192,0.03997759819030762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,128,0.004853333532810211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,16384,0.07581120332082113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,6144,0.03145066698392232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,16384,0.06592640082041422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,16384,0.1428885300954183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,51200,0.1816970666249593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,12288,0.09439679781595865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,12288,0.05922773281733194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,10240,0.0789514700571696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,10240,0.05174933274586996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,8192,0.06416746775309244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,10240,0.041360000769297287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,8192,0.0340992013613383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,8192,0.04413119951883952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,7168,0.05335466861724854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,7168,0.04054400126139323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,7168,0.03061973253885905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,6144,0.046855465571085615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,6144,0.02725226680437724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,6144,0.03758506774902344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,5120,0.03315946658452352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,4096,0.03186666568120321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,3072,0.018566399812698364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,4096,0.02993280092875163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,4096,0.020644267400105797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,3584,0.029121067126591998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,3584,0.027770666281382243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,3072,0.02533866763114929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,3072,0.02574933369954427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,2560,0.020959999163945517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,2560,0.023710934321085612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,2560,0.014615466197331747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,2048,0.017262933651606242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,2048,0.022006400426228843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,2048,0.012558933099110922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,1536,0.013653332988421122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,1536,0.020267732938130698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,1536,0.010597333312034607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,51200,0.2007370630900065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,1024,0.010201600193977357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,12288,0.048207998275756836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,1024,0.01925333340962728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,768,0.0092330664396286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,768,0.016685867309570314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,512,0.0066655998428662615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,512,0.014791466792424521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,512,0.005772800246874491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,256,0.003605333218971888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,5120,0.023900800943374635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,256,0.015009066462516785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,256,0.005107200145721436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,128,0.003328000009059906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,128,0.014734933773676554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,128,0.004729599754015604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,64,0.0029311999678611755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,32,0.002993066608905792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,64,0.014849066734313965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,32,0.014713600277900696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,65536,0.19958292643229167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,65536,0.37361812591552734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,65536,0.18669546445210775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,51200,0.1596768061319987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,51200,0.29446185429890953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,5120,0.039204267660776775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,16384,0.09830079873402914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,16384,0.06864319642384847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,16384,0.05376960039138794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,12288,0.07038613160451254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,51200,0.14571626981099445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,12288,0.04953920046488444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,10240,0.05882026751836141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,12288,0.039817599455515544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,10240,0.04707839886347453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,10240,0.034179198741912845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,8192,0.04782613515853882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,7168,0.04226346810658772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,7168,0.03522026538848877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,3584,0.01881813406944275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,7168,0.025524266560872394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,6144,0.03735466798146565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,5120,0.031344000498453775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,6144,0.0324234664440155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,3072,0.016251732905705772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,5120,0.01998186707496643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,5120,0.029487999280293782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,4096,0.025590399901072185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,3584,0.022881066799163817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,4096,0.017614932854970296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,4096,0.026234666506449383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,3072,0.02026346723238627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,3584,0.01590826710065206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,3584,0.02477226654688517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,2560,0.01722986698150635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,3072,0.013938132921854654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,3072,0.023331199089686075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,2560,0.01260693371295929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,2560,0.02206719915072123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,2048,0.020361600319544475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,1024,0.008707200487454731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,1536,0.011431466539700825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,1536,0.019301333030064902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,1536,0.00962559978167216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,768,0.007086933155854543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,1024,0.008550399541854858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,1024,0.007975466549396515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,1024,0.01726933320363363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,768,0.0073088000218073535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,8192,0.039301331837972006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,768,0.014912000298500061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,768,0.006088533500830332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,512,0.005713066458702088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,256,0.003487999985615412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,512,0.00561599979797999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,512,0.01527679959932963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,256,0.0051466668645540874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,128,0.0031221332649389905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,256,0.0150218665599823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,64,0.002890666574239731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,128,0.004811733464399974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,128,0.014668800433476768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,32,0.00297173336148262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,64,0.014739200472831726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,32,0.014685866236686707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,2048,0.01432213286558787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,65536,0.1815839926401774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,65536,0.16249279975891112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,65536,0.3255743980407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,51200,0.14626986185709637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,16384,0.10273173650105794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,51200,0.2574656009674072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,51200,0.12890346844991046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,16384,0.05666559934616089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,16384,0.0457535982131958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,12288,0.06555519898732504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,12288,0.03592106501261393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,10240,0.054467201232910156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,10240,0.041381335258483885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,10240,0.03086613416671753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,8192,0.04442453384399414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,8192,0.025616000096003216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,8192,0.03492586612701416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,7168,0.03969706694285075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,6144,0.03515626589457194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,7168,0.033004800478617355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,6144,0.03046506643295288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,5120,0.029785599311192828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,5120,0.027900799115498857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,5120,0.018151466051737467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,8192,0.0291594664255778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,4096,0.02526719967524211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,6144,0.023242666323979696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,3584,0.02076266606648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,3584,0.02372373342514038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,2048,0.010854400197664897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,12288,0.04650880098342895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,3072,0.0182751993338267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,3072,0.0227018674214681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,2560,0.015650133291880287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,3072,0.012779733538627625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,2560,0.021356799205144248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,2048,0.013126400113105775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,2048,0.010249599814414978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,2048,0.019989333550135293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,1536,0.010734933614730834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,7168,0.023218133052190146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,1536,0.01880319913228353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,1536,0.009152000149091084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,6144,0.021011199553807577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,1024,0.006770133475462596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,1024,0.015987199544906617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,768,0.006838400165239971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,512,0.004036266605059306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,768,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,512,0.0056202664971351625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,256,0.0034400001168251038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,4096,0.023229867219924927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,256,0.014889599879582724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,128,0.003156266609827677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,128,0.014806399742762247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,128,0.004902400076389313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,64,0.0029088000456492106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,64,0.014572800199190775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,32,0.002918400118748347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,32,0.014665599664052328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,65536,0.2800170580546061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,1024,0.007986133297284443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,65536,0.15560107231140136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,512,0.015106133619944253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,65536,0.14279252688090008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,51200,0.12519360383351644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,51200,0.22092480659484864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,16384,0.07475732962290446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,16384,0.057532799243927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,16384,0.0401802659034729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,12288,0.057300265630086264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,12288,0.04277439912160237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,4096,0.015890133380889893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,10240,0.048981332778930665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,3584,0.014115200440088908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,10240,0.038045867284138994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,10240,0.027448532978693645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,8192,0.03963626623153686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,8192,0.03354133367538452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,7168,0.03489813407262166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,7168,0.03139839967091878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,7168,0.020849066972732543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,6144,0.030443733930587767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,2560,0.011710932850837708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,6144,0.029000532627105714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,6144,0.01871573328971863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,5120,0.02582293351491292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,5120,0.02643306652704875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,5120,0.01630826691786448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,4096,0.021215999126434328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,4096,0.02424853245417277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,3584,0.019246933857599895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,768,0.0060917332768440245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,3584,0.02301759918530782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,3584,0.013051733374595642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,3072,0.016910932461420693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,3072,0.02204586664835612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,3072,0.0116074671347936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,256,0.005202133456865946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,2560,0.020457599560419717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,2560,0.01083733340104421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,2048,0.012350933750470479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,2048,0.009546666344006857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,2048,0.01938986579577128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,1536,0.009914666414260864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,1536,0.018150399128595986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,1536,0.00865280032157898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,1024,0.007412266731262207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,1024,0.015290666619936624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,1024,0.006379733482996623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,768,0.006343466540177663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,768,0.015406933426856995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,512,0.003939199944337209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,512,0.015064533551534018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,512,0.005458133419354757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,256,0.003304533412059148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,256,0.014961066842079162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,2560,0.014521599809328715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,128,0.003052799900372823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,128,0.014677332838376364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,128,0.004659200211366018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,51200,0.11362026532491047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,64,0.0028746667007605235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,32,0.0029002666473388673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,64,0.0144405335187912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,32,0.014536533753077188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,12288,0.03213013410568237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,65536,0.13640106519063314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,65536,0.23807573318481445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,65536,0.12337600390116374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,8192,0.02285333275794983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,51200,0.10643306573232014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,16384,0.06318186521530152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,51200,0.1878485361735026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,16384,0.04516160090764364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,12288,0.04843200047810872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,12288,0.02833706736564636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,12288,0.03741546471913655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,10240,0.04121706485748291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,10240,0.033268266916275026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,10240,0.024563199281692503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,8192,0.03347200155258179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,8192,0.029491200049718218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,7168,0.028042666117350262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,7168,0.027764266729354857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,7168,0.019006933768590292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,6144,0.02480640014012655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,4096,0.013848533233006796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,6144,0.025988266865412397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,6144,0.01680533289909363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,5120,0.024124799172083537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,5120,0.014656000336011252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,4096,0.017425066232681273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,4096,0.022206934293111165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,3584,0.015740799903869628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,3584,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,3584,0.012098133563995361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,3072,0.013966932892799377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,3072,0.020399999618530274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,3072,0.010829866925875346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,2560,0.012045866250991822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,768,0.005957333246866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,2560,0.019621332486470543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,2560,0.01025493343671163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,2048,0.018569600582122803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,2048,0.00888320008913676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,16384,0.03509333531061808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,1536,0.008654933174451191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,1536,0.007904000083605449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,1024,0.006660266717274983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,1536,0.01685973405838013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,256,0.004982399940490723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,1024,0.006309333443641663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,1024,0.015336533387502035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,768,0.005445333321889242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,768,0.015149866541226705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,768,0.005843199789524078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,512,0.003717333326737086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,512,0.014758400122324624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,256,0.0032469332218170166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,5120,0.021409066518147786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,256,0.004913066824277243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,128,0.0029685333371162414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,4096,0.012705066800117492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,128,0.014519466956456503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,128,0.004785066843032837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,64,0.002775466690460841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,32,0.0028586665789286296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,64,0.014310399691263834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,32,0.014298666516939798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,65536,0.186407470703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,65536,0.108787202835083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,2048,0.010250666737556457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,51200,0.1483850638071696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,51200,0.08791146278381348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,16384,0.04955519835154216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,51200,0.08379840056101481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,16384,0.03842560052871704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,51200,0.0986303965250651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,16384,0.030408533414204915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,12288,0.038865065574645995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,12288,0.02451200087865194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,10240,0.03296746611595154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,12288,0.03311040004094441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,10240,0.021627734104792275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,10240,0.030108799537022907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,256,0.014600533246994018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,8192,0.020488532384236653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,8192,0.017730132738749186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,8192,0.02672106623649597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,7168,0.024445867538452147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,6144,0.02175253431002299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,7168,0.02512213389078776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,6144,0.023588265975316366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,5120,0.018949333826700845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,6144,0.014938666423161825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,4096,0.01573013365268707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,5120,0.01325866679350535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,5120,0.022293333212534586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,3584,0.013125333189964294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,4096,0.02086506684621175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,3584,0.020091732343037925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,3584,0.01123520036538442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,3072,0.011767466862996418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,3072,0.019409066438674925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,3072,0.010220799843470256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,2560,0.010305066903432209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,2560,0.018771199385325114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,2560,0.009698133667310078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,2048,0.00890773336092631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,2048,0.008353066444396973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,2048,0.017698132991790773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,1536,0.0075573335091273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,1536,0.01532906691233317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,1024,0.005826133489608765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,1024,0.015200000007947287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,8192,0.027074132363001508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,768,0.003961600114901861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,768,0.015080533425013223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,7168,0.01640959978103638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,768,0.005611733098824819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,512,0.005354666709899902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,512,0.005218133330345154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,512,0.01498240033785502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,256,0.0032000000278155005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,256,0.014851199587186179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,256,0.004890666902065277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,128,0.0029440000653266907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,128,0.0046858668327331545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,64,0.002807466685771942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,128,0.014578133821487427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,32,0.002735999971628189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,64,0.014459733168284097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,32,0.014454399545987448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,65536,0.16805119514465333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,65536,0.10283520221710205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,51200,0.08194560209910075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,51200,0.13099413712819416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,65536,0.1068277359008789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,16384,0.047990401585896805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,51200,0.08211519718170165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,16384,0.035813331604003906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,16384,0.030025599400202434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,12288,0.03469226757685344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,12288,0.030826665957768756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,12288,0.024195200204849242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,10240,0.02982826630274455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,10240,0.028410667181015016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,8192,0.024536534150441488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,10240,0.021064533789952596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,512,0.0034847999612490333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,7168,0.022375466426213582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,8192,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,7168,0.024167466163635253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,6144,0.01986453334490458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,6144,0.02296853264172872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,6144,0.01395093301932017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,5120,0.01723626653353373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,4096,0.011805867155392964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,5120,0.021497599283854165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,5120,0.013040000200271606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,4096,0.013366400202115377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,4096,0.02053119937578837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,4096,0.011108266313870747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,3584,0.012061867117881774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,3584,0.019717333714167277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,3584,0.011047466595967611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,3072,0.010966400305430096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,3072,0.019143466154734293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,2560,0.009781333804130554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,2560,0.018467199802398682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,2560,0.00944106678167979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,2048,0.008307200173536937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,2048,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,1536,0.007298133273919423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,2048,0.007630933324495952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,1536,0.007165866593519847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,1536,0.007108266651630402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,1536,0.015412267049153647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,1024,0.0047199999292691554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,1024,0.01527679959932963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,1024,0.006196266909440359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,768,0.0038261334101359046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,768,0.015056000153223673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,768,0.005619200070699056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,512,0.0035637333989143372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,7168,0.016309332847595216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,512,0.014905599753061929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,512,0.005404800176620483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,256,0.014539733529090881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,128,0.0029418667157491045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,128,0.01432319978872935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,65536,0.09898880322774252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,64,0.0027456000447273255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,64,0.014512000481287637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,32,0.00276799996693929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,32,0.014452266693115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,65536,0.1434549331665039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,1024,0.00600853314002355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,65536,0.08702932993570964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,256,0.003176533430814743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,51200,0.07099839846293131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,51200,0.11338773568471272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,16384,0.03948266506195068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,16384,0.03347733418146769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,51200,0.07544746398925781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,16384,0.027905066808064777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,12288,0.031446399291356404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,12288,0.02883946696917216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,10240,0.02658453385035197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,12288,0.022449066241582237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,8192,0.021887999773025513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,10240,0.019550933440526327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,10240,0.02632319927215576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,7168,0.019782400131225585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,8192,0.016520532965660095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,8192,0.024778666098912557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,6144,0.017356799046198527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,7168,0.015314132968584696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,7168,0.023612799247105916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,5120,0.015260799725850423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,6144,0.013931733369827271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,6144,0.02211839954058329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,4096,0.013769599795341491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,5120,0.01245013376077016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,5120,0.02092693249384562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,4096,0.019451733430226645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,3584,0.012628266215324402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,3584,0.0108106662829717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,3584,0.018999467293421425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,3072,0.0113045334815979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,3072,0.018541866540908815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,2560,0.008938666184743245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,3072,0.009732266267140705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,2560,0.017717333634694417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,2560,0.008242133259773254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,2048,0.007761066655317943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,2048,0.007473066449165344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,2048,0.01583999991416931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,1536,0.006681600213050842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,1536,0.01562879979610443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,1536,0.006770133475462596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,1024,0.004347733159859975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,1024,0.015265066425005594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,1024,0.005743999779224396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,768,0.0038560000558694207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,768,0.015121066570281982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,768,0.005543466905752818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,512,0.003432533393303553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,512,0.014968533317248026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,512,0.0052490666508674625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,256,0.005021866659323374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,256,0.0030986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,256,0.004931200047334035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,8192,0.017480534315109254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,128,0.002937600016593933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,128,0.004626133541266123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,128,0.014670933286348978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,64,0.0026709333062171934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,32,0.0027797333896160126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,64,0.014383999506632486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,32,0.014551466703414917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,65536,0.11967360178629557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,65536,0.07568000157674154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,128,0.004796800017356872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,51200,0.06043306589126587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,51200,0.0955082654953003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,16384,0.03322346607844035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,51200,0.074235733350118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,16384,0.03060479958852132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,16384,0.027539199590682982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,12288,0.02602880001068115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,12288,0.026409600178400678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,12288,0.02213866710662842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,10240,0.02267199953397115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,10240,0.024668800830841064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,10240,0.019350399573644005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,8192,0.01894186735153198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,8192,0.022555732727050783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,8192,0.016456533471743265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,7168,0.017127466201782227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,7168,0.021865600347518922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,6144,0.015427199999491372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,6144,0.021053866545359293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,256,0.014646400014559427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,6144,0.013758933544158936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,5120,0.013596799969673157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,5120,0.020155733823776244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,65536,0.09438933531443278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,4096,0.011734400192896526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,4096,0.019157334168752035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,3584,0.010961066683133442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,3584,0.018832000096638997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,3584,0.0105813334385554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,3072,0.010154666503270467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,3072,0.01742186745007833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,4096,0.01095146636168162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,3072,0.009639466802279156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,2560,0.008302933474381765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,2560,0.008547199765841167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,2560,0.016017066438992818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,2048,0.007092266778151194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,2048,0.016106667121251424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,2048,0.007245866457621257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,1536,0.005767466624577841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,1536,0.015643733739852905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,1536,0.0069461335738499955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,1024,0.004147200038035711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,1024,0.015197867155075073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,768,0.0038506666819254553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,768,0.014967466394106546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,768,0.005666133264700572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,512,0.003435733417669932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,512,0.014816000064214071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,512,0.005328000088532766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,256,0.00315733328461647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,256,0.014602667093276978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,256,0.004991999765237173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,128,0.0028522667785485585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,128,0.01455573340257009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,128,0.004655999938646952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,64,0.00276799996693929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,32,0.0027882667879263563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,64,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,32,0.014556800325711569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,65536,0.0929866631825765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,65536,0.09619519710540772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,65536,0.06173973480860392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,51200,0.07468266487121582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,51200,0.051215998331705724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,16384,0.027170133590698243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,51200,0.06805439790089926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,16384,0.026953599850336712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,16384,0.02533866763114929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,12288,0.021630932887395225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,12288,0.02416320045789083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,10240,0.018820265928904213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,7168,0.015146666765213012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,10240,0.022957867383956908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,8192,0.015481600165367126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,10240,0.01789120038350423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,8192,0.01552959978580475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,8192,0.02146453261375427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,7168,0.014200533429781595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,7168,0.02056000034014384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,7168,0.014284800489743552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,6144,0.012758400042851767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,5120,0.011573333541552227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,6144,0.013117866714795432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,5120,0.011743999520937602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,4096,0.010942932963371278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,5120,0.019500799973805747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,4096,0.010357333223025005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,4096,0.01850559910138448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,4096,0.010384000341097514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,3584,0.00957973301410675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,3584,0.017193599541982015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,3584,0.009769599636395771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,3072,0.009320533275604248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,3072,0.008962133526802063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,3072,0.008826667070388794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,3072,0.016340266664822897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,2560,0.00766186664501826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,2560,0.008094933132330577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,2560,0.016616533199946083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,2048,0.0065290664633115125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,2048,0.015842133760452272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,2048,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,1536,0.00469760000705719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,1536,0.015658666690190635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,1024,0.004113066693147024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,1024,0.015198933084805808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,1024,0.005762133498986562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,768,0.003689600030581156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,768,0.015310933192571005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,1024,0.005946666498978933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,512,0.0033503999312718712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,6144,0.019722666343053183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,512,0.015034666657447815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,256,0.003050666550795237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,256,0.014722133676211039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,256,0.004806399842103322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,128,0.0028512001037597655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,128,0.014526933431625366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,128,0.004645333190759023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,64,0.002647466709216436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,64,0.014545067151387533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,32,0.0026496000587940215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,32,0.014597333470980325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,65536,0.07377599875132243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,65536,0.051269332567850746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,65536,0.08527466456095377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,51200,0.05890880028406779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,51200,0.043277867635091144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,12288,0.020670932531356812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,16384,0.021824000279108684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,16384,0.024267733097076416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,16384,0.024589866399765015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,12288,0.01788160006205241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,12288,0.02199466625849406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,12288,0.019834667444229126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,10240,0.015286399920781454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,5120,0.011453866958618164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,10240,0.021061333020528157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,10240,0.017417599757512413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,8192,0.013019733627637229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,8192,0.014979199568430582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,8192,0.019976532459259032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,7168,0.012011733651161195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,7168,0.01950613260269165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,6144,0.011158399780591329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,6144,0.018837332725524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,5120,0.010078932841618855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,5120,0.01898026665051778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,65536,0.08372053305308023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,5120,0.010984533031781514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,4096,0.008946133653322856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,1536,0.006572799881299336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,4096,0.009397332866986591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,3584,0.008520533641179402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,3584,0.009178666273752849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,3584,0.016474666198094685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,3072,0.008006399869918824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,3072,0.016166399916013083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,3072,0.008096000055472057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,2560,0.007096533477306366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,2560,0.016275200247764587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,2560,0.007621333499749501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,2048,0.005358933409055074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,2048,0.015631999572118124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,2048,0.0067445332805315655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,1536,0.004650666813055674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,768,0.005406933526198069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,1536,0.006455466647942861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,1536,0.01548373301823934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,1024,0.004043733328580856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,1024,0.0056991999348004665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,768,0.0037450666228930154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,1024,0.01503679951032003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,512,0.0033919999996821085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,768,0.0054400001962979635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,768,0.015049599607785544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,256,0.0030623999734719592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,512,0.005027199784914652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,512,0.005144533514976501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,128,0.002762666592995326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,256,0.014654933412869772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,128,0.01437226633230845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,64,0.0026133333643277483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,64,0.014446933070818582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,32,0.0026687999566396077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,32,0.014517333110173544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,4096,0.01753173271814982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,65536,0.04930560191472371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,65536,0.039443198839823404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,51200,0.06645439863204956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,51200,0.03985919952392578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,51200,0.03389546473821004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,512,0.014751999576886495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,256,0.004867200056711833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,16384,0.01571626663208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,51200,0.06634879906972249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,128,0.004698666433493296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,16384,0.021886932849884033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,12288,0.01997119983037313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,12288,0.019761067628860474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,10240,0.011891200145085653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,10240,0.017437867323557534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,10240,0.019418666760126747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,8192,0.010334933797518413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,8192,0.018433066209157307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,8192,0.0150218665599823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,7168,0.009673600395520527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,7168,0.013833600282669067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,7168,0.018133334318796792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,6144,0.008756267031033833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,6144,0.01732800006866455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,6144,0.012005333105723064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,5120,0.008398933211962382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,5120,0.017871999740600587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,5120,0.010545066992441813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,4096,0.0075882668296496075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,4096,0.017321600516637167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,4096,0.009241599837938945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,3584,0.006974933544794719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,6144,0.012563199798266093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,3584,0.00893440047899882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,3072,0.0064650664726893115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,3072,0.016539733608563742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,2560,0.006123733520507812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,2560,0.016145066420237223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,2560,0.007751466830571492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,2048,0.005409066875775656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,2048,0.015692800283432007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,2048,0.0068256000677744556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,1536,0.004750933249791463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,1536,0.015507200360298156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,12288,0.013202133774757385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,1536,0.006461866696675618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,1024,0.004051200052102407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,1024,0.005690666536490122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,768,0.0036170666416486105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,768,0.0149536003669103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,768,0.005248000224431356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,512,0.0033503999312718712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,512,0.014878933628400167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,256,0.0030239999294281008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,3584,0.016713599363962807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,256,0.004903466502825419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,256,0.014775466918945313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,3072,0.008056533336639405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,128,0.014705066879590353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,64,0.003020799905061722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,64,0.014731733004252115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,32,0.0029728000362714132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,32,0.014812800288200378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,65536,0.03858879804611206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,65536,0.08417173226674399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,65536,0.03459626833597819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,51200,0.031429332494735715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,1024,0.015056000153223673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,51200,0.030180267492930096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,16384,0.013103999694188437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,51200,0.06669440269470214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,16384,0.02032426595687866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,16384,0.024345600605010988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,12288,0.011410133043924967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,12288,0.019618133703867592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,12288,0.01874133348464966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,10240,0.01032533347606659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,10240,0.016847999890645345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,8192,0.009305600325266521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,8192,0.016936532656351724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,7168,0.013517866532007853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,8192,0.01442026694615682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,7168,0.008558932940165203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,7168,0.017672532796859743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,7168,0.013240533073743186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,6144,0.007806933422883351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,128,0.0028885332246621448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,6144,0.01172693371772766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,6144,0.017107200622558594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,5120,0.007179733117421467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,5120,0.01053653359413147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,5120,0.017875200510025023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,4096,0.009261866410573322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,4096,0.017273600896199545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,3584,0.006623999774456024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,3584,0.016809600591659545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,3072,0.006078933179378509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,3072,0.016376533110936484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,3072,0.007982933521270752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,10240,0.018151466051737467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,2560,0.0059456000725428265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,2560,0.01618346671263377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,512,0.005009066561857859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,2048,0.005302399893601735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,128,0.004645333190759023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,4096,0.00613013356924057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,2048,0.01595946649710337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,1536,0.004666666686534882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,2048,0.006774400174617767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,1024,0.004030933231115341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,1536,0.006495999793211619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,3584,0.008906666437784832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,1024,0.0056757330894470215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,1024,0.014964266618092855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,768,0.0036330667634805044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,512,0.003409066547950109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,768,0.005447466671466827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,512,0.0050346667567888895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,512,0.014973866939544677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,256,0.00317546675602595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,256,0.014857600132624308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,256,0.004880000154177348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,128,0.002926933268706004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,128,0.014284800489743552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,128,0.004680533210436503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,64,0.0028192001084486645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,32,0.002757333219051361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,64,0.0145087997118632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,65536,0.027342933416366576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,32,0.014457600315411887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,65536,0.08407466411590576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,51200,0.022651733954747517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,65536,0.027730133136113482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,65536,0.08315093517303467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,51200,0.025144533316294355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,16384,0.010468266407648722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,16384,0.018797866503397622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,16384,0.02408746679623922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,12288,0.009168000022570292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,12288,0.01760853330294291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,12288,0.019621332486470543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,10240,0.008685866991678875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,16384,0.024718934297561647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,10240,0.01819093426068624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,10240,0.01660053332646688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,8192,0.007730133334795634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,7168,0.006713599960009258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,8192,0.013949867089589438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,8192,0.017492266496022542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,6144,0.006167466441790262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,7168,0.012844799955685934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,6144,0.011622400085131327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,6144,0.017553067207336424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,5120,0.0064735998709996535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,5120,0.01815999945004781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,5120,0.010481066505114238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,4096,0.005961599946022034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,1536,0.015524267156918844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,4096,0.009249066313107807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,3584,0.006309333443641663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,3584,0.008840533097585042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,768,0.015065600474675497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,3072,0.0164682666460673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,3072,0.008027733365694682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,2560,0.006074666480223338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,2560,0.01629973351955414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,2560,0.007547733187675476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,2048,0.00544213354587555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,2048,0.006843733290831248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,2048,0.015929599603017174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,1536,0.0047647997736930845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,1536,0.006313600142796834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,1024,0.004172799984614054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,1024,0.015170133113861084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,1024,0.005713066458702088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,768,0.003851733356714249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,768,0.014868266383806863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,7168,0.017794134219487508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,768,0.005332266787687937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,512,0.0034143999218940735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,512,0.005054933329423269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,512,0.014683733383814493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,256,0.0031797334551811217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,256,0.004714666803677877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,2560,0.007579733431339264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,256,0.0145578662554423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,4096,0.017322667439778647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,128,0.0028618666032950084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,128,0.004588800172011057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,64,0.0027306665976842242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,128,0.014455466469128927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,3584,0.016952532529830932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,64,0.014648532867431641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,3072,0.005983999868233999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,65536,0.017130666971206666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,32,0.014311466614405313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,65536,0.022741333643595377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,51200,0.014334932963053385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,65536,0.08331946531931558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,16384,0.0075573335091273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,51200,0.02111146648724874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,12288,0.006257066627343495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,16384,0.017678932348887125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,12288,0.017123200496037803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,10240,0.006162133316198985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,10240,0.017494400342305504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,10240,0.01620266636212667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,8192,0.006045866509278615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,8192,0.017378133535385133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,1536,0.015634133418401083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,8192,0.01390720009803772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,7168,0.005858133236567179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,7168,0.017554134130477905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,7168,0.012928000092506409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,6144,0.005806933343410492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,6144,0.011524266997973124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,6144,0.01749546726544698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,5120,0.006233599781990051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,5120,0.01036906639734904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,5120,0.017921066284179686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,4096,0.005779199798901876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,4096,0.01726933320363363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,4096,0.009170132875442504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,3584,0.0063360000650088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,3584,0.016756266355514526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,3584,0.00885653297106425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,32,0.00277866671482722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,3072,0.016495999693870545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,2560,0.006099199752012888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,3072,0.008029866715272267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,2560,0.00749120016892751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,2560,0.01602133313814799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,16384,0.02392746607462565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,2048,0.005340800185998281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,1536,0.0047199999292691554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,2048,0.01576746702194214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,12288,0.01888213356335958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,2048,0.006735999882221222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,1536,0.006303999821345012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,1024,0.003995733211437861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,1536,0.015356799960136414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,1024,0.01527253290017446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,768,0.003636266787846883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,1024,0.005625600119431814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,768,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,768,0.005241600175698599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,512,0.014762666821479798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,256,0.0030432000756263735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,512,0.005043200155099233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,128,0.002885333448648453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,256,0.014481066664059957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,128,0.004574933151404063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,128,0.014382933576901754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,64,0.0027029333015282948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,32,0.0027061333258946735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,64,0.014473600188891092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,65536,0.01176106631755829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,32,0.01434879998366038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,65536,0.019802665710449217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,51200,0.06606080134709677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,51200,0.010321066776911417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,65536,0.08254506587982177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,51200,0.018984532356262206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,51200,0.06496533155441284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,16384,0.017357865969340004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,16384,0.023388799031575522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,12288,0.005767466624577841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,12288,0.017230933904647826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,12288,0.0187008003393809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,10240,0.0060597335298856105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,3072,0.005974400043487549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,8192,0.005919999877611796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,10240,0.016313599546750386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,8192,0.01685973405838013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,7168,0.005771733323733012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,7168,0.0172160009543101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,7168,0.012699733177820841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,6144,0.00557226687669754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,6144,0.016857600212097167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,512,0.0032853332658608755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,6144,0.011518933375676473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,5120,0.005987200140953064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,51200,0.06588799953460693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,256,0.004752000172932943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,5120,0.01730666756629944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,5120,0.01033066709836324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,4096,0.00913706620534261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,3584,0.0060032000144322716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,4096,0.016720000902811685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,3584,0.016756266355514526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,3072,0.005710933109124502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,3584,0.008821333448092144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,2560,0.005621333420276642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,3072,0.007905066510041555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,3072,0.01634986698627472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,2560,0.015890133380889893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,2048,0.005100800096988678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,2560,0.007428266604741414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,1536,0.004531200230121613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,2048,0.00678719977537791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,1536,0.015335466464360556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,1024,0.003770666569471359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,16384,0.005964800218741099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,1024,0.014991999665896098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,1024,0.005673600236574808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,768,0.0035904000202814737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,768,0.005278933544953664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,768,0.014900267124176025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,512,0.0033002667129039764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,512,0.004930133124192556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,256,0.002997333308060964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,10240,0.017564799388249716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,256,0.01458026667435964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,256,0.004741333425045013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,128,0.002812800059715907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,128,0.004479999840259552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,128,0.014296533664067588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,64,0.0026496000587940215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,32,0.0026890667776266735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,64,0.014465066790580749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,65536,0.008846933643023174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,32,0.014315733313560485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,51200,0.0076789334416389465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,65536,0.018620800971984864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,16384,0.00576853354771932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,51200,0.018004266421000163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,12288,0.005683200061321258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,16384,0.017169066270192466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,10240,0.005881600081920624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,12288,0.016858667135238647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,4096,0.0056320001681645715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,10240,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,7168,0.005636266867319743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,8192,0.01681173245112101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,6144,0.005598933498064677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,7168,0.017369600137074788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,5120,0.005919999877611796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,6144,0.017072000106175742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,2048,0.015473066767056783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,4096,0.005533866584300995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,5120,0.017298134167989095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,1536,0.006285866598288219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,4096,0.016716800133387246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,3072,0.005718400080998739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,3584,0.016727467377980552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,2560,0.005701333284378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,3072,0.016603733102480568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,2048,0.005031466484069824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,2560,0.015947733322779337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,1536,0.004439466694990794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,2048,0.015479466319084168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,1024,0.0038176000118255613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,1536,0.0151829332113266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,768,0.0034517332911491393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,1024,0.014885333180427552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,512,0.0031360000371932983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,768,0.014964266618092855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,256,0.0029663999875386557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,512,0.01474666694800059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,512,0.014680533607800802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,128,0.002809600035349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,256,0.014654933412869772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,64,0.002682666728893916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,128,0.014350933829943338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,32,0.002648533384005229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,64,0.014293332894643148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,65536,0.007945600152015685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,32,0.014281599720319112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,65536,0.018663465976715088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,16384,0.005710933109124502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,51200,0.01825066606203715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,12288,0.005584000051021576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,16384,0.017114667097727458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,10240,0.005806933343410492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,12288,0.01691626707712809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,8192,0.005749333401521047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,10240,0.017283199230829875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,7168,0.00561599979797999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,8192,0.016666666666666666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,6144,0.005545599758625031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,7168,0.017484800020853678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,5120,0.005913599828879039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,8192,0.013853866855303446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,6144,0.017114667097727458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,4096,0.005500799914201101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,5120,0.017340799172719322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,3584,0.00595413347085317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,4096,0.016939733425776163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,3584,0.016315733393033348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,2560,0.005612800021966299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,3072,0.016218666235605875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,2048,0.005070933202902476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,2560,0.015878400206565856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,1536,0.004491733511288961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,2048,0.01581013302008311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,1024,0.0038730666041374207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,1536,0.015153066317240397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,768,0.00348693331082662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,1024,0.014967466394106546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,512,0.0032458665470282235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,768,0.01480959951877594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,256,0.0029056000212828318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,512,0.014648532867431641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,128,0.0027434666951497394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,256,0.014318933089574179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,3584,0.006028800209363302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,64,0.0026858667532602947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,128,0.014434132973353067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,32,0.0026367999613285064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,64,0.01418346663316091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,32,0.014476799964904785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,51200,0.007433600227038066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,3072,0.0056650668382644655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,8192,0.005757866799831391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,16384,0.7452725092569987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,16384,0.39030186335245765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,16384,0.3570591926574707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,12288,0.26823787689208983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,12288,0.5639349619547527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,51200,1.1789386749267579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,10240,0.24842774073282875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,10240,0.23102720578511557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,10240,0.48077014287312825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,8192,0.1817728042602539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,8192,0.20569067001342772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,7168,0.32900587717692054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,8192,0.42896000544230145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,7168,0.17922239303588866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,7168,0.1600159962972005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,51200,2.3563306172688803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,6144,0.2836693445841471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,6144,0.1749695936838786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,12288,0.2963605244954427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,5120,0.13247786362965902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,5120,0.2363189379374186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,4096,0.19247786204020184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,4096,0.10849066575368245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,3584,0.1667978604634603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,4096,0.09322026570638021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,3584,0.09785280227661133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,5120,0.13547520637512206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,3584,0.09448320070902506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,3072,0.14375680287679035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,2560,0.1203722635904948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,3072,0.07314346631368002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,3072,0.08621333440144857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,2560,0.08444159825642904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,2048,0.09616639614105224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,1536,0.07409493128458658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,2048,0.06239253282546997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,1536,0.05740053256352743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,1536,0.03946239948272705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,1024,0.053091200192769375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,1024,0.039146665732065836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,768,0.039318398634592695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,1024,0.03139839967091878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,768,0.033613868554433185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,512,0.027011199792226152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,768,0.022566399971644082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,256,0.015153066317240397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,512,0.029080533981323244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,256,0.020933334032694498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,128,0.010052266716957092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,128,0.020357332626978555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,64,0.006248533229033152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,64,0.01871573328971863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,65536,32,0.004537599782148996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,65536,32,0.018630399306615194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,51200,1.094048055013021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,6144,0.16069973309834798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,2560,0.06082666714986166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,2048,0.05011946757634481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,512,0.01679253379503886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,256,0.011929600437482198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,65536,1.2052448272705079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,65536,128,0.009351467092831928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,65536,2.3533342997233073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,16384,0.2950058619181315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,16384,0.3701247851053874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,16384,0.599882698059082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,65536,1.1522144317626952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,12288,0.5269237200419108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,12288,0.23899520238240562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,12288,0.22367253303527831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,10240,0.37321173350016273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,51200,0.9503402709960938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,10240,0.21770453453063965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,8192,0.16860267321268718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,10240,0.22915199597676597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,8192,0.30646613438924153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,51200,0.8941685358683268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,8192,0.1638805389404297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,7168,0.3013333320617676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,7168,0.14349973996480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,6144,0.12651946544647216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,51200,1.8495445251464844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,6144,0.13324586550394696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,6144,0.22437334060668945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,5120,0.18670719464619953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,5120,0.09623573621114095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,4096,0.14812053044637044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,3584,0.1422752062479655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,3584,0.06951893170674642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,3584,0.0793066660563151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,3072,0.11336106459299725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,3072,0.07057493527730306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,2560,0.09505813121795655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,2560,0.060482132434844973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,2048,0.07784319718678792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,2048,0.05218559900919596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,2048,0.042372266451517746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,1536,0.05923733313878378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,1536,0.04696213404337565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,7168,0.1329578638076782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,1536,0.03754133383433024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,1024,0.04543040196100871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,1024,0.03388373454411824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,1024,0.024116265773773193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,768,0.030204800764719646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,5120,0.11931733290354411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,768,0.01986879905064901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,768,0.028467200199762982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,512,0.021637332439422608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,4096,0.10011626879374187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,512,0.015086932977040609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,512,0.023848533630371094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,128,0.00766293356815974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,256,0.011024000247319539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,128,0.015497600038846334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,256,0.019964800278345744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,64,0.004003199934959412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,128,0.00788373351097107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,32,0.004088533421357473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,32,0.015796266992886863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,51200,64,0.015233066678047181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,65536,0.3778250694274902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,65536,0.7487658818562826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,51200,256,0.012222933769226074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,4096,0.07790613174438477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,16384,0.10648852984110516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,16384,0.18447359402974445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,51200,0.3066261291503906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,3072,0.059918932120005285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,51200,0.580668830871582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,16384,0.09370240370432535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,51200,0.2783498764038086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,12288,0.14039467175801595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,12288,0.07177706559499106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,10240,0.07235413392384847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,12288,0.09685226281483969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,8192,0.0940661350886027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,10240,0.1377130667368571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,8192,0.06123413244883219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,8192,0.04957226514816284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,7168,0.08304639657338461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,6144,0.07207146485646566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,7168,0.044061867396036784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,6144,0.049700268109639484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,5120,0.06024959882100424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,5120,0.03321920037269592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,5120,0.04479786554972331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,4096,0.04887679815292358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,4096,0.038023467858632407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,3584,0.04319786628087362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,3584,0.03548159996668498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,3072,0.03743893305460612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,3072,0.032580266396204635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,3072,0.022037333250045775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,2560,0.03173440098762512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,2560,0.029639466603597002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,2560,0.01978666583697001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,2048,0.02584853370984395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,2048,0.026331732670466106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,2048,0.016152532895406087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,1536,0.02047146757443746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,1536,0.02339306672414144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,1536,0.013541332880655923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,10240,0.0706816037495931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,1024,0.014408533771832785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,1024,0.01043839951356252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,1024,0.020563199122746786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,768,0.011562666296958924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,7168,0.05519040028254191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,768,0.01919893423716227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,512,0.008661333719889324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,65536,0.35476481119791664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,512,0.017563732465108235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,256,0.005995733539263407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,256,0.01532266636689504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,4096,0.027453867594401042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,128,0.003504000107447306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,128,0.00486826648314794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,128,0.014805333813031516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,64,0.0031360000371932983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,64,0.01474346617857615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,16384,32,0.003189333279927572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,16384,32,0.014697600404421488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,65536,0.29167359670003257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,65536,0.552294413248698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,768,0.009239466985066731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,512,0.007549866537253062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,65536,0.28931732177734376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,6144,0.038634665807088214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,51200,0.23246933619181315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,3584,0.025121066967646283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,51200,0.43393920262654617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,51200,0.22464000384012855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,16384,0.07581760088602701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,16384,0.08624853293100992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,12288,0.1128992001215617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,12288,0.07042240301767985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,12288,0.05696213245391846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,10240,0.09195199807484945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,10240,0.05772906541824341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,10240,0.04862399895985921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,8192,0.07499840259552001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,8192,0.04991039832433065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,8192,0.03959146738052368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,7168,0.06609493494033813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,7168,0.04580906629562378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,51200,2560,0.05129066705703735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,6144,0.055027198791503903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,6144,0.040278398990631105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,6144,0.03164479931195577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,5120,0.046011734008789065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,5120,0.036497068405151364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,5120,0.02727893392244975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,4096,0.022832000255584718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,3584,0.032671999931335446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,4096,0.03495466709136963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,3584,0.03041279911994934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,3072,0.02918826738993327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,3072,0.027782400449117024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,2560,0.024834134181340537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,2560,0.025337600708007814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,2560,0.01637440025806427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,2048,0.02037013371785482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,2048,0.02336960037549337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,16384,0.14434986114501952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,16384,256,0.005400533477465311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,1536,0.017950934171676636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,1536,0.021691733598709108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,1024,0.01172693371772766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,1024,0.01923946738243103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,1024,0.009310932954152425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,768,0.009827199578285217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,768,0.01808533271153768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,768,0.008158933122952778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,512,0.007652266820271809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,7168,0.03601706822713216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,512,0.015507200360298156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,512,0.005814399818579356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,4096,0.036714665095011395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,256,0.005224533379077911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,256,0.015146666765213012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,128,0.0035242666800816857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,128,0.014811733365058899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,128,0.004814933240413666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,64,0.0031786667803923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,3072,0.018574933211008705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,32,0.0031199999153614043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,64,0.014813866217931113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,12288,32,0.014906666676203408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,65536,0.4736202557881673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,65536,0.2345109303792318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,65536,0.24795734087626137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,51200,0.3702698707580566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,16384,0.12340479691823322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,16384,0.07565759817759196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,16384,0.06569279829661051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,51200,0.1910645325978597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,51200,0.199343999226888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,12288,0.09383359750111898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,12288,0.05914880037307739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,12288,0.055813332398732506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,10240,0.05146773258845011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,10240,0.08177279631296794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,8192,0.06417920192082724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,8192,0.04413439830144246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,7168,0.05334399938583374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,8192,0.03543146848678589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,7168,0.04062399864196777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,7168,0.030473599831263225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,6144,0.04672319889068603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,3584,0.021082667509714763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,6144,0.03694506486256917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,6144,0.02763200004895528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,5120,0.039214932918548585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,5120,0.03322986761728923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,5120,0.024029866854349772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,4096,0.03166293303171794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,4096,0.029176533222198486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,3584,0.02823786735534668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,4096,0.020483199755350748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,3584,0.027590399980545043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,3584,0.018619734048843383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,3072,0.02555519938468933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,2560,0.020971733331680297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,2560,0.014478933811187745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,2560,0.023962666591008507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,2048,0.01732586622238159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,2048,0.022064000368118286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,2048,0.013803733388582864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,2048,0.012612266341845193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,1536,0.013641599814097086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,1536,0.020451200008392335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,1536,0.010732799768447876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,1024,0.010085333387056987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,1024,0.008617599805196125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,1024,0.018665599822998046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,12288,256,0.0041514667371908825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,768,0.016709333658218382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,512,0.006695466736952464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,512,0.015098667144775391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,3072,0.02464853326479594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,3072,0.016390400131543477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,256,0.003585066646337509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,256,0.005133866767088572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,256,0.0150026669104894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,128,0.003260799994071325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,128,0.004756266872088114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,128,0.014682666460673014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,64,0.003011200080315272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,64,0.014625066518783569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,32,0.002914133419593175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,12288,1536,0.011720533172289532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,10240,32,0.014734933773676554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,10240,768,0.00844586690266927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,65536,0.19974826176961263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,65536,0.37207892735799153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,65536,0.1838538646697998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,51200,0.2945301373799642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,16384,0.09264853000640869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,10240,0.04241600036621094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,16384,0.06031466722488403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,16384,0.05317866802215576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,12288,0.07014933427174887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,51200,0.15974292755126954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,12288,0.04949866533279419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,10240,0.05883200168609619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,10240,0.04342079957326253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,8192,0.047725868225097653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,8192,0.0383296012878418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,7168,0.04231573343276977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,7168,0.03510186672210693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,6144,0.03724160194396973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,6144,0.032264532645543416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,6144,0.023166932662328086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,5120,0.03128746747970581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,5120,0.029307732979456585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,5120,0.020197333892186482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,4096,0.02545493245124817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,4096,0.02608213424682617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,4096,0.01733013391494751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,512,0.005801600217819214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,3584,0.022808533906936646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,3584,0.015569067001342774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,3584,0.02476373314857483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,3072,0.020147200425465903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,3072,0.023246934016545616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,3072,0.013871999581654868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,2560,0.017204266786575318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,2560,0.022055466969807945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,2560,0.012794666488965354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,2048,0.014353066682815552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,2048,0.020388267437616982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,1536,0.011498666803042094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,1536,0.019132800896962485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,1536,0.009701333443323771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,1024,0.008594133456548055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,1024,0.01722453236579895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,1024,0.007411199808120728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,768,0.00726506660381953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,768,0.015267200271288552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,768,0.006035199761390686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,512,0.005690666536490122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,10240,768,0.00705813318490982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,51200,0.14675520261128744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,256,0.0035413332283496858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,12288,0.03985386689503988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,2048,0.011245866616566975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,256,0.01518933375676473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,128,0.00317546675602595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,10240,0.034279465675354004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,8192,0.0281056006749471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,128,0.0047978664437929785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,64,0.0030271999537944795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,8192,32,0.003010133405526479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,64,0.0146506667137146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,32,0.014678399761517844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,7168,0.025650133689244587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,65536,0.16319573720296224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,65536,0.325873057047526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,65536,0.18031360308329264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,512,0.015256533026695251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,16384,0.05633493264516195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,16384,0.0886133352915446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,51200,0.14606614112854005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,51200,0.25676053365071616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,12288,0.06532266537348429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,12288,0.04650986591974894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,16384,0.048002131779988605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,10240,0.054934398333231604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,12288,0.03595626751581828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,10240,0.04118613402048747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,8192,0.04435520172119141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,10240,0.030580266316731768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,8192,0.035792001088460285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,7168,0.03952213525772095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,8192,0.025421865781148273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,7168,0.03293226758639018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,6144,0.03491946856180827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,7168,0.023124267657597862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,6144,0.0303872009118398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,5120,0.02982613245646159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,6144,0.020985599358876547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,256,0.005048533280690512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,4096,0.022964266935984294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,5120,0.027852799495061236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,8192,128,0.014792533715566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,3584,0.020709333817164104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,4096,0.015621333320935567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,3072,0.0183242658774058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,3584,0.014291200041770934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,3584,0.023884799083073935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,3072,0.012683733304341634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,3072,0.022478934129079184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,2560,0.015864533185958863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,2560,0.011740799744923909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,2560,0.021393066644668578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,2048,0.013154133160909017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,1536,0.010788266857465107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,2048,0.01027413308620453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,2048,0.0197653333346049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,1024,0.008094933132330577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,1536,0.009102933605511983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,1536,0.018846933046976724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,1024,0.006855466465155284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,1024,0.015473066767056783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,768,0.006157866617043813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,768,0.015387733777364096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,512,0.004131199916203817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,512,0.015291733543078103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,512,0.00561706672112147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,256,0.0034133332471052804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,256,0.014856533209482829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,256,0.005143466591835022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,128,0.0030901332696278887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,128,0.004882133503754934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,128,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,64,0.0028512001037597655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,32,0.0028607999285062153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,64,0.014628266294797262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,8192,512,0.005818666517734527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,32,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,65536,0.15512213706970215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,65536,0.14232640266418456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,7168,4096,0.0248416006565094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,51200,0.12467626730600993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,51200,0.2208394686381022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,51200,0.11334400177001953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,16384,0.07460693518320718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,16384,0.0547104001045227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,7168,768,0.006877866884072621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,12288,0.0571232000986735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,12288,0.042290135224660234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,10240,0.04901653528213501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,12288,0.031983999411265056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,10240,0.03747733434041341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,10240,0.027280000845591228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,8192,0.03944106499354045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,8192,0.03377279837926229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,7168,0.034962133566538496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,7168,0.031251200040181476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,7168,0.020866133769353232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,6144,0.030393600463867188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,6144,0.029330132404963176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,6144,0.018708266814549766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,5120,0.02558079957962036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,5120,0.02648746569951375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,5120,0.016220800081888833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,4096,0.021572266022364298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,4096,0.024253867069880166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,51200,0.1335968017578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,4096,0.01397119959195455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,3584,0.019186133146286012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,3072,0.01696746746699015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,3584,0.012972799936930337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,3584,0.024526933828989662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,65536,0.2794848124186198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,3072,0.011811199784278869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,2560,0.01448319951693217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,2560,0.010916266838709514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,2560,0.020644267400105797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,2048,0.01954986651738485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,2048,0.009431466460227966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,1536,0.01000426709651947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,7168,5120,0.018101332585016887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,1536,0.018294399976730345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,1024,0.007451733450094859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,1024,0.00639573335647583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,768,0.006376533210277558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,1024,0.01581013302008311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,768,0.00556160012880961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,768,0.015277866522471109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,512,0.0039264000952243805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,512,0.015339733163515726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,512,0.005264000097910563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,256,0.003399466723203659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,256,0.014781866470972696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,256,0.004990933338801066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,128,0.0030613332986831666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,8192,0.022819199164708457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,128,0.0046634669105211895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,128,0.014634666840235391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,32,0.0028394666810830434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,64,0.014648532867431641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,32,0.014665599664052328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,65536,0.13586026827494305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,65536,0.23720426559448243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,6144,3072,0.021871999899546305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,65536,0.12328960100809734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,2048,0.012409599622090657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,51200,0.18748586972554523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,16384,0.06277439991633096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,51200,0.10676053365071614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,51200,0.09826773007710775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,16384,0.035554134845733644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,12288,0.04838613271713257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,12288,0.03703573147455851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,10240,0.040873599052429196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,10240,0.03342399994532268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,10240,0.024489599466323852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,8192,0.02949013312657674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,8192,0.020507733027140297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,7168,0.02795093258221944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,16384,0.04045120080312093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,6144,64,0.0028885332246621448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,7168,0.027784534295399982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,7168,0.01876373291015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,6144,0.024615466594696045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,6144,0.025949867566426595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,6144,0.016603733102480568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,5120,0.021082667509714763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,5120,0.014694399634997048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,4096,0.02232426603635152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,4096,0.012709333499272665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,3584,0.015809067090352378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,3584,0.02145706613858541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,16384,0.04837546745936076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,12288,0.028164267539978027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,3072,0.020606933037439983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,3072,0.010822400450706482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,8192,0.03343146642049154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,2560,0.01220373312632243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,6144,1536,0.008795733253161114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,2560,0.01961173415184021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,2048,0.01032960017522176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,2048,0.018567466735839845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,2048,0.008963200449943542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,1536,0.008556800087292989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,1536,0.0077898666262626644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,1024,0.006764799853165944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,1024,0.015166933337847391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,1024,0.006413866579532623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,5120,0.0240064005057017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,768,0.0054848000407218935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,768,0.005842133363087972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,768,0.015273599823315939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,512,0.0037941334148248037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,512,0.005438933273156484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,256,0.0032149332265059153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,256,0.014703999956448874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,256,0.0050240000089009605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,128,0.0030847998956839246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,128,0.014393599828084311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,128,0.004877866804599762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,64,0.0028661333024501802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,64,0.014457600315411887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,3072,0.013926399747530618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,32,0.0028789333999156954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,32,0.014572800199190775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,65536,0.1076586643854777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,65536,0.18625599543253582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,65536,0.10538986523946126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,1536,0.017220266660054526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,51200,0.08764053185780843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,51200,0.0837440013885498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,16384,0.04987733364105225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,16384,0.03853226502736409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,12288,0.03868159850438436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,16384,0.03036160071690877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,12288,0.033078400293986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,12288,0.024800000588099162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,10240,0.032884265979131064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,8192,0.027266132831573486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,5120,512,0.01497066617012024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,8192,0.02718186577161153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,7168,0.024730666478474935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,8192,0.01812160015106201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,7168,0.024961066246032716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,7168,0.016539733608563742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,6144,0.021695999304453532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,6144,0.023552000522613525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,3584,0.012082133690516155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,5120,0.019041067361831664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,5120,0.022323199113210044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,5120,0.013274666666984559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,4096,0.015901866555213928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,4096,0.020835200945536293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,4096,0.011645866433779399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,3584,0.013245866696039835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,51200,0.14728426933288574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,3584,0.020165334145228066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,3584,0.011213866869608562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,3072,0.019463467597961425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,5120,2560,0.010268800457318624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,2560,0.010380799571673077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,5120,4096,0.01750613252321879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,2560,0.01890453298886617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,10240,0.021717333793640138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,2048,0.00888213316599528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,2048,0.017858133713404337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,1536,0.007652266820271809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,1536,0.015228799978892007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,1536,0.007115733126799266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,1024,0.005869866907596588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,1024,0.015396266182263692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,1024,0.005961599946022034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,768,0.003979733337958654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,768,0.01511573294798533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,6144,0.014905599753061929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,512,0.0035904000202814737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,512,0.014759467045466105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,3072,0.011850666999816895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,256,0.003171200056870779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,256,0.01483519971370697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,256,0.004866133133570353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,2560,0.009664000074068705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,128,0.0028970666229724885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,10240,0.03012266755104065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,128,0.004675200084845225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,64,0.0028053333361943563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,128,0.014542933305104574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,4096,32,0.002811733384927114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,64,0.014588800072669984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,4096,32,0.014562132954597472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,65536,0.16756265958150227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,65536,0.10277653535207112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,51200,0.13105599880218505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,51200,0.08158613046010335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,16384,0.047082666556040445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,768,0.005599999924500784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,16384,0.03573866685231526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,16384,0.029975465933481854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,12288,0.034753068288167314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,10240,0.029616000254948933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,12288,0.024100265900293984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,12288,0.030801065762837726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,8192,0.02443199952443441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,10240,0.028437334299087524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,8192,0.0176256000995636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,8192,0.025415466229120894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,7168,0.022163200378417968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,7168,0.024228266874949136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,7168,0.01627306640148163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,6144,0.019474132855733236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,6144,0.022805333137512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,6144,0.014593066771825156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,5120,0.017153066396713258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,5120,0.021661865711212158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,4096,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,4096,0.020307199160257975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,4096,0.011668266852696736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,3584,0.012076800068219502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,3584,0.019740800062815346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,65536,0.0992469310760498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,3072,0.010920533537864685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,3072,0.010132267077763876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,2048,0.008273066580295562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,2560,0.009634133179982502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,3072,0.009964799880981446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,3072,0.019045333067576088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,2048,0.008278400202592214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,2560,0.00942080020904541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,2560,0.01836586594581604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,51200,0.08166186809539795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,1536,0.00717439999183019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,2048,0.007804800073305767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,2048,0.016474666198094685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,1024,0.0046741331617037455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,1536,0.007009066641330719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,1536,0.015636266271273295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,1024,0.015412267049153647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,768,0.0038901334007581077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,1024,0.0061258668700853985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,512,0.003519999980926514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,768,0.005704533557097117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,768,0.01515733301639557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,256,0.003134933362404505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,512,0.0150026669104894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,512,0.00535999983549118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,128,0.002961066613594691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,256,0.004930133124192556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,256,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,128,0.014648532867431641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,64,0.0027200000981489818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3584,32,0.0027200000981489818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,64,0.014666666587193808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3584,32,0.014652799566586813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,4096,512,0.005274666845798493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,65536,0.1431455930074056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,5120,0.01308799982070923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,65536,0.08661333719889322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,65536,0.0939903974533081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,51200,0.07088853518168131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,51200,0.11294079621632894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,51200,0.07527360121409098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,16384,0.039468801021575926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,12288,0.03129066626230876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,16384,0.033321599165598556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,16384,0.030789333581924438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,12288,0.028658133745193482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,10240,0.02648319999376933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,12288,0.02264960010846456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,10240,0.02653333346048991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,8192,0.02190293272336324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,8192,0.024804266293843587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,7168,0.019725867112477622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,8192,0.01644373337427775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,6144,0.017427200078964235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,7168,0.015238400300343832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,7168,0.023436800638834635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,5120,0.01535146633783976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,6144,0.013906133174896241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,6144,0.022412800788879396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,5120,0.01244160036245982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,4096,0.013703466455141703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,5120,0.020897066593170165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,3584,0.012526933352152506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,4096,0.019568000237147012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,3584,0.010520533720652262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,3584,0.0191103994846344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,3072,0.011425066987673442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,3072,0.018618667125701906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,3072,0.009508267045021057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,2560,0.008943999807039898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,2560,0.0086816002925237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,2560,0.017366399367650352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,2048,0.00773119976123174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,2048,0.015962666273117064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,2048,0.007525333265463512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,128,0.004726399978001913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,1536,0.0066549330949783325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,1024,0.004369066655635833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,1536,0.0067456002036730455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,1536,0.015719466408093772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,768,0.003953066716591517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,1024,0.005961599946022034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,1024,0.015299200018246969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,512,0.0035445332527160645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,768,0.015253333250681558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,10240,0.021082667509714763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,512,0.01490133305390676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,256,0.00315733328461647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,256,0.01482133368651072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,128,0.0028629332780838014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,128,0.014517333110173544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,128,0.004732800026734671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,64,0.0026975999275843303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,10240,0.019691733519236247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,64,0.014503467082977294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,3072,32,0.0026922665536403658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,3072,32,0.014450132846832275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,65536,0.07534933090209961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,65536,0.11930773258209229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,4096,0.010870400071144103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3584,3584,0.011020800471305848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,51200,0.09476693471272787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,65536,0.09309546947479248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,51200,0.06018346548080444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,16384,0.03427520195643107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,16384,0.03033813238143921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,12288,0.02600853244463603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,12288,0.02624853253364563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,10240,0.02258666753768921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,10240,0.019297067324320474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,10240,0.024462932348251344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,8192,0.019001599152882895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,8192,0.02251840035120646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,8192,0.016330666343371072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,7168,0.01707306702931722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,7168,0.02184106707572937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,7168,0.015122133493423461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,768,0.005564799904823304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,6144,0.015291733543078103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,6144,0.02094399929046631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,5120,0.020423465967178346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,256,0.0050453335046768185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,5120,0.012342400352160136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,4096,0.011657599608103435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,4096,0.018908800681432088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,5120,0.013482667009035745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,3584,0.010808533430099488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,3584,0.01862293283144633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,3072,512,0.005306666592756907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,3072,0.00995306670665741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,3072,0.01771519978841146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,3072,0.009270399808883667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,2560,0.00817920019229253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,2560,0.01665066679318746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,2560,0.008575999736785888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,2048,0.007175466914971669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,2048,0.015904000401496886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,51200,0.07411733468373617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,16384,0.02555946707725525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,1536,0.00588266650835673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,12288,0.0221450666586558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,1024,0.004114133367935816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,1536,0.006540800134340922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,1536,0.01562879979610443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,1024,0.005801600217819214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,768,0.003833599885304769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,1024,0.01539413332939148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,512,0.0034517332911491393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,768,0.005509333312511444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,768,0.015270400047302245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,256,0.003235200047492981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,512,0.005123200019200643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,512,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,256,0.004922666649023692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,128,0.0028533334533373516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,256,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,64,0.0027850667635599775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,128,0.004738133152325948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,128,0.014579199751218162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2560,32,0.002791466563940048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,64,0.014503467082977294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2560,32,0.01446293294429779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,65536,0.09601386388142905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,65536,0.06112213134765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,4096,0.010877866546312969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,65536,0.08373653093973796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,51200,0.05114346742630005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,16384,0.027074132363001508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,51200,0.068067200978597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,16384,0.026814933617909747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,12288,0.021755733092625937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,12288,0.020549333095550536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,12288,0.02399253249168396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,10240,0.01893333395322164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,10240,0.01810879906018575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,10240,0.022696532805760703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,8192,0.015434666474660238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,8192,0.02121280034383138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,7168,0.014270933469136557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,7168,0.020517333348592123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,7168,0.014355199535687766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,6144,0.01284160017967224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,6144,0.01307093302408854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,6144,0.019975467522939046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,5120,0.011659733454386393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,5120,0.019509333372116088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,5120,0.011735467116038005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,4096,0.010193066795667012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,4096,0.01865066687266032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,4096,0.009970133503278095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,3584,0.009581866860389709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,3584,0.016668800512949625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,3584,0.009796266754468281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,3072,0.008853333195050557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,3072,0.008644266923268636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,3072,0.01629866659641266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,6144,0.013873066504796347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,2560,0.015833600362141927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,2048,0.006741333504517872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,3584,0.010359467069307963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,2048,0.015698132912317912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,1536,0.004730666677157084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,51200,0.07473066647847494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,1536,0.015447466572125753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,1536,0.006451199948787689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,1024,0.0040554667512575785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,1024,0.015135999520619711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,768,0.003730133424202601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,768,0.014876799782117209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,768,0.0054400001962979635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,8192,0.015594666202863058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,512,0.0034175999462604523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,512,0.0050805335243542995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,512,0.014847999811172486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,256,0.0030239999294281008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,256,0.004990933338801066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2560,2048,0.007229866584142049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,128,0.002855466554562251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,128,0.014694399634997048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,64,0.002664533257484436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,64,0.014550399780273438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,32,0.00262719988822937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,2048,2560,0.007609599828720092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,2560,0.007830399771531422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,32,0.014664533734321594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,65536,0.07371093432108561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,2048,0.007077333331108093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,65536,0.05129813353220621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,51200,0.058879999319712314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,65536,0.08371413548787435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,51200,0.04312853415807088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,16384,0.021970132986704506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,51200,0.06641066471735636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,16384,0.024280534187952677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,12288,0.01775040030479431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,16384,0.024562132358551026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,10240,0.015629866719245912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,12288,0.019806933403015137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,12288,0.022167466084162393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,8192,0.013132799665133157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,10240,0.021166932582855225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,8192,0.015003732840220132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,8192,0.01994133392969767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,7168,0.01225493351618449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,7168,0.0195360004901886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,7168,0.01390720009803772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,6144,0.011128532886505126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,6144,0.018898133436838785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,5120,0.010154666503270467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,5120,0.018757333358128868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,16384,0.025066665808359784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,1024,0.0055637334783871974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,4096,0.008975999553998311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,2048,256,0.014550399780273438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,4096,0.009322667121887207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,4096,0.01688746611277262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,3584,0.008609066406885784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,3584,0.008909866213798523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,3072,0.00803413341442744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,3584,0.016966400543848674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,2560,0.006780800223350525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,3072,0.01629866659641266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,2560,0.007717333237330119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,2560,0.016082132856051125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,2048,0.005354666709899902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,2048,0.015824000040690102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,1536,0.004752000172932943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,2048,128,0.0045503998796145115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,1536,0.015565866231918335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,1536,0.006405333181222279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,1024,0.004081066697835922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,1024,0.005724800129731497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,1024,0.01513706644376119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,768,0.0036703998843828833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,768,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,512,0.003316266586383184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,512,0.014876799782117209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,512,0.005043200155099233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,256,0.0030154667794704436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,256,0.004849066833655039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,256,0.014662399888038635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,128,0.0028522667785485585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,128,0.014532267053922018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,64,0.002631466587384542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,64,0.014443733294804893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1536,32,0.0026911998788515727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1536,32,0.014447999993960061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,65536,0.049449598789215087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,65536,0.039009066422780354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,10240,0.017437867323557534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,3072,0.008148266871770223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,65536,0.08336426417032877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,51200,0.03965973456700643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,16384,0.01585813363393148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,51200,0.06622506777445475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,2048,0.006807466844717662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,16384,0.02460800011952718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,6144,0.012635733683904013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,12288,0.013301333785057068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,12288,0.019845332702000937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,10240,0.011974400281906128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,12288,0.01971199909845988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,8192,0.010286933183670044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,10240,0.0195360004901886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,5120,0.011131733655929565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,8192,0.014706133802731832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,8192,0.01800533334414164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,7168,0.009763200084368389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,7168,0.013571199774742127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,7168,0.017638399203618368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,6144,0.008769067128499348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,5120,0.008335999647776286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,6144,0.01717653274536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,5120,0.017960532506306966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,5120,0.010505599776903789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,4096,0.007492266595363617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,4096,0.01714986761411031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,4096,0.009278933207194012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,3584,0.007157333195209503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,128,0.004649599889914194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,3584,0.016753067572911583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,3072,0.01652479966481527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,51200,0.03421866496404012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,3072,0.008059733112653096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,16384,0.021425066391626994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,2560,0.00602346658706665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,2560,0.016266666849454246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,2048,0.005415466427803039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,10240,0.01737706661224365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,2048,0.01584106683731079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,2048,0.0068010667959849044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,1536,0.004752000172932943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,1536,0.0063381334145863845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,1536,0.01551253298918406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,1024,0.004011733333269755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,1024,0.005658666789531708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,1024,0.015156267086664834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,768,0.0037109332780043284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,768,0.014936533570289613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,768,0.005317333340644837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,512,0.003399466723203659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,512,0.015058133006095886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,512,0.0050911997755368555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,256,0.0029845332105954488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,3072,0.006324266890684764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,256,0.014472533265749613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,256,0.0047989333669344585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,128,0.004738133152325948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,128,0.014674133062362671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,64,0.0029696000119050344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,32,0.0029546665648619336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,64,0.014806399742762247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,1024,32,0.014893866578737893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,65536,0.03881066640218099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,65536,0.034017066160837814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,51200,0.03131733338038127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,65536,0.08337066968282064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,51200,0.03001813292503357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,51200,0.06614293257395426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,16384,0.01316373348236084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,16384,0.024554665883382162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,16384,0.02002560098965963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,12288,0.01134399970372518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,12288,0.01856213410695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,12288,0.019644800821940103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,2560,0.0076330666740735366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,10240,0.010351999600728353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,8192,0.009276800354321798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,8192,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,8192,0.014189866185188294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,7168,0.008664533495903015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,7168,0.017563732465108235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,7168,0.01301653285821279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,6144,0.007794133325417836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,6144,0.017413334051767985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1536,768,0.005382399757703145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,6144,0.011587199568748475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,6144,0.011826133728027344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,1024,128,0.002807466685771942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,4096,0.0061034664511680605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,5120,0.010484266281127929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,5120,0.017966934045155845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,3584,0.00655680000782013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,4096,0.00921066701412201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,4096,0.0172106663386027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,3072,0.006067200005054474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,3584,0.008827733000119527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,3584,0.017189333836237587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,3072,0.007977599898974102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,2560,0.006082133452097575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,3072,0.016547200083732606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,2048,0.005406933526198069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,2560,0.007660800218582153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,2560,0.01624853312969208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,1536,0.004626133541266123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,2048,0.006695466736952464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,2048,0.015898666779200234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,1024,0.004033066580692927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,1536,0.006414933502674103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,1536,0.015533866484959922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,768,0.0035968000690142312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,1024,0.015260799725850423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,768,0.005346133311589559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,512,0.0033258666594823206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,512,0.015003732840220132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,256,0.003172266731659571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,512,0.004930133124192556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,10240,0.018170666694641114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,128,0.002963199963172277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,256,0.014813866217931113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,128,0.0046304002404212955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,128,0.014667733510335287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,64,0.002833066632350286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,32,0.0028394666810830434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,64,0.014694399634997048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,65536,0.027296000719070436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,32,0.01455893317858378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,65536,0.02770986755688985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,51200,0.022605866193771362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,1024,3584,0.008804266651471455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,51200,0.02482346693674723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,768,5120,0.007316266496976216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,16384,0.010398933291435241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,16384,0.018849066893259683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,16384,0.02412160038948059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,12288,0.009100799759229023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,12288,0.01733760039011637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,12288,0.019230933984120686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,10240,0.0084906667470932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,10240,0.017449599504470826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,10240,0.01676266590754191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,768,768,0.01578986644744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,8192,0.007620266576608021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,8192,0.01723840037981669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,7168,0.006636799871921539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,7168,0.01789439916610718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,10240,0.01678933302561442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,7168,0.012939733266830445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,6144,0.006138666470845541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,6144,0.01738133430480957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,5120,0.006250666578610738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,5120,0.018080000082651773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,5120,0.010396800438563029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,4096,0.005824000140031179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,4096,0.017364267508188883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,4096,0.009160533547401428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,3584,0.006400000055631001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,3584,0.01694719990094503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,3584,0.008899199962615966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,3072,0.006045866509278615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,3072,0.016696532567342125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,3072,0.007921066880226136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,2560,0.006167466441790262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,2560,0.01620693306128184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,2560,0.007625600198904674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,2048,0.00547733356555303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,2048,0.01588159998257955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,1024,0.005703466633955637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,1536,0.004791466891765595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,1536,0.01567359964052836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,1536,0.006295466423034668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,1024,0.004107733319203059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,1024,0.015286399920781454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,768,256,0.004844800134499868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,1024,0.0056085333228111265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,768,0.003869866579771042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,768,0.0052597333987553915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,768,0.015228799978892007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,512,0.0034602666894594826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,512,0.004991999765237173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,256,0.0031317333380381264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,256,0.014615466197331747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,256,0.004774400095144907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,128,0.0029077333708604175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,65536,0.08304746945699057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,128,0.014434132973353067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,51200,0.06589333216349283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,64,0.0027637332677841187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,512,32,0.0026975999275843303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,64,0.014503467082977294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,32,0.014417066176732381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,65536,0.016784000396728515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,51200,0.01420906682809194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,65536,0.022549333175023396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,65536,0.08280426661173503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,51200,0.021041067441304524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,16384,0.007367466886838277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,51200,0.06568426688512166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,16384,0.023695999383926393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,16384,0.017927465836207072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,12288,0.006287999947865804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,10240,0.006231466432412466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,12288,0.018782933553059898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,10240,0.017633066574732462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,8192,0.006002133091290792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,8192,0.01723519961039225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,8192,0.014039466778437296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,7168,0.005957333246866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,7168,0.017656532923380534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,7168,0.012760532895723977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,6144,0.011632000406583149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,6144,0.005890133480230967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,6144,0.01744426687558492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,5120,0.006298666695753734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,512,512,0.014775466918945313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,5120,0.018001067638397216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,5120,0.010393599669138592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,4096,0.005866666634877523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,4096,0.009215999643007915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,4096,0.01729173262914022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,3584,0.00625600020090739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,3584,0.016876800855000814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,3584,0.008794666330019633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,3072,0.005994666616121928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,3072,0.01648640036582947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,3072,0.007904000083605449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,2560,0.006089599927266439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,2560,0.00751146674156189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,2048,0.0053269331653912864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,2048,0.015943466623624166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,2048,0.006748799979686737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,1536,0.0047647997736930845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,12288,0.01733120083808899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,10240,0.016404267152150473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,1536,0.015470932920773825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,1024,0.003974399964014689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,2048,0.006760533154010773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,1024,0.01509119967619578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,768,0.003642666588226954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,1024,0.005493333439032236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,768,0.005474133292833964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,768,0.014774399995803832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,512,0.0032960000137488045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,512,0.014738133549690247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,512,0.004973866542180379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,256,0.002940800040960312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,256,0.014516266187032065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,6144,0.011471999684969585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,256,0.004731733103593191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,128,0.0028213332096735638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,128,0.004598399996757508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,128,0.014413866400718688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,64,0.002679466704527537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,256,32,0.002681600054105123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,64,0.014290133118629455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,65536,0.011406933267911274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,32,0.0143477330605189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,65536,0.019090133905410766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,65536,0.08230400085449219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,51200,0.017941333850224814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,16384,0.005876266459623972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,16384,0.017217065890630087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,16384,0.023549866676330567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,12288,0.005890133480230967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,256,2560,0.016089600324630738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,12288,0.017117865880330405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,12288,0.018644267320632936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,10240,0.0061247999469439185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,10240,0.01637440025806427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,8192,0.005946666498978933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,512,128,0.004644266764322917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,8192,0.016871466239293417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,8192,0.013893333077430726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,7168,0.00581333339214325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,1536,0.006333866715431213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,7168,0.01275200049082438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,6144,0.0057536001006762184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,6144,0.016986666123072307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,6144,0.011468799908955891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,5120,0.0059349333246548975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,5120,0.010391466816266378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,5120,0.01758613387743632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,4096,0.0055637334783871974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,4096,0.01689599951108297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,4096,0.009153067072232565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,3584,0.006118399898211161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,3584,0.016578132907549538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,256,8192,0.014032000303268432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,3072,0.005739733576774597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,3072,0.007893333335717519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,3072,0.016247466206550598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,2560,0.005672533313433329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,2560,0.016042666633923848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,2560,0.007551999886830647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,51200,0.010025599598884582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,2048,0.01565120021502177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,2048,0.006662400066852569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,1536,0.004546133180459341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,1536,0.015548800428708395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,1536,0.006291200220584869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,1024,0.003857066730658213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,1024,0.015166933337847391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,1024,0.005490133166313171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,768,0.003636266787846883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,768,0.015140266219774882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,512,0.003319466610749563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,512,0.014968533317248026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,512,0.004861866434415182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,10240,0.017249067624409996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,256,0.003058133274316788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,256,0.0047775998711586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,256,0.014521599809328715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,128,0.0028031999866167706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,128,0.00452693353096644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,128,0.014407466848691305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,64,0.0027093333502610523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,7168,0.01729173262914022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,32,0.00262719988822937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,64,0.01428053379058838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,65536,0.008689066767692566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,51200,0.0074869334697723385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,65536,0.01815999945004781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,128,32,0.014524799585342408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,16384,0.00574186642964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,51200,0.017887999614079796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,12288,0.005677866439024607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,16384,0.017118932803471883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,12288,0.01707093318303426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,8192,0.005769599974155426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,10240,0.01727466583251953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,3584,0.00874773363272349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,7168,0.0056874667604764305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,8192,0.017010132471720375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,6144,0.005580799778302511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,7168,0.017220266660054526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,5120,0.0061152001221974695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,6144,0.016965333620707193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,4096,0.0055871998270352686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,5120,0.017338667313257852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,4096,0.016820265849431356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,3584,0.006062933305899302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,3072,0.005700266857941946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,3584,0.0162282665570577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,2560,0.0056085333228111265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,3072,0.016110933820406594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,2048,0.005021866659323374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,2560,0.015677866339683533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,1536,0.004489600161711375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,2048,0.015426133076349893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,1024,0.003853866706291834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,1536,0.015241600076357522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,768,0.0035018667578697203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,1024,0.014822399616241455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,512,0.003160533308982849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,768,0.014666666587193808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,256,0.003009066730737686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,512,0.014596266547838846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,128,0.0027850667635599775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,256,0.014455466469128927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,64,0.0026133333643277483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,128,0.014371200402577718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,32,0.002630399912595749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,51200,0.0648629347483317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,64,0.014375467101732889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,65536,0.00772159993648529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,64,32,0.01420906682809194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,51200,0.006842666864395141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,65536,0.01793173352877299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,16384,0.005891199906667074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,51200,0.01829013427098592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,12288,0.005718400080998739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,16384,0.017288533846537273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,12288,0.01694399913152059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,8192,0.005758933226267497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,10240,0.017130666971206666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,7168,0.005640533566474914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,8192,0.01676693360010783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,6144,0.005543466905752818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,7168,0.017347200711568197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,5120,0.005899733304977417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,6144,0.017146666844685875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,4096,0.005481599768002828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,5120,0.017569067080815633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,3584,0.005962666869163513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,4096,0.016693333784739174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,3072,0.005604266623655955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,3584,0.016378666957219443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,2560,0.005655466516812643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,3072,0.016059733430544534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,2048,0.0050335998336474095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,2560,0.015847466389338174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,1536,0.004423466821511587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,1024,0.0037450666228930154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,1536,0.01525973379611969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,768,0.003522133330504099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,1024,0.014726400375366211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,512,0.0031850665807724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,768,0.014516266187032065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,256,0.0028938665986061097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,512,0.014428800344467163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,128,0.002735999971628189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,128,2048,0.0051146666208903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,256,0.014243200421333313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,64,0.0025759999950726825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,128,0.01425386667251587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,32,0.0026378666361172995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,32,0.014313600460688271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,32,10240,0.00582826683918635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,3,128,768,0.00528959979613622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,3,64,10240,0.005875200033187866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,2048,0.015500799814860026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,3,32,64,0.014169599612553915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,16384,0.7460394541422526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,16384,0.3900501251220703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,16384,0.3552032152811686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,12288,0.5652650833129883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,12288,0.3034901301066081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,51200,1.1880906422932944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,12288,0.27096853256225584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,10240,0.47228161493937176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,10240,0.2489173412322998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,8192,0.3764927864074707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,8192,0.17938027381896973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,10240,0.22601920763651528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,8192,0.23847039540608725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,7168,0.32864745457967126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,7168,0.17884480158487956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,51200,2.33743158976237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,7168,0.17854080200195313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,6144,0.15576640764872235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,6144,0.13751573562622071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,6144,0.28351573944091796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,5120,0.13240746657053631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,5120,0.2448117256164551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,4096,0.19197440147399902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,4096,0.09260906378428141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,4096,0.10821653207143148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,3584,0.1668063958485921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,5120,0.13515733083089193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,3584,0.10212480227152507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,3584,0.08827733198801677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,3072,0.08591787020365396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,3072,0.14369279543558758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,3072,0.07123839855194092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,2560,0.07406293551127116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,2560,0.06033066511154175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,2560,0.14116586049397786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,2048,0.09615039825439453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,2048,0.06172053416570028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,1536,0.0738090674082438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,1024,0.038571735223134354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,1024,0.050236801306406655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,1536,0.04421439965565999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,1536,0.05113919973373413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,768,0.03900479873021444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,512,0.026796799898147584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,768,0.03354560136795044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,768,0.02513386607170105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,512,0.026637866099675493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,256,0.015095466375350952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,512,0.016502400239308677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,128,0.009009066224098205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,256,0.02207146684328715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,128,0.020053333044052123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,128,0.00916373332341512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,32,0.004364799956480662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,64,0.006817066669464111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,64,0.018615466356277467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,32,0.018771199385325114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,2048,0.049864534536997476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,51200,1.0954303741455078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,65536,1.1983168284098307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,65536,1.1533973693847657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,1024,0.02880000074704488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,16384,0.3673386573791504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,256,0.01195733348528544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,65536,2.3472000122070313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,16384,0.5979818979899089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,51200,0.9433834711710611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,51200,1.8446389516194661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,12288,0.2355445384979248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,16384,0.29141546885172526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,12288,0.4507744153340657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,51200,0.9080970764160157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,10240,0.3714399973551432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,10240,0.21475839614868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,12288,0.25344640413920083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,8192,0.16110612551371256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,10240,0.21829546292622887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,8192,0.14969493548075358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,8192,0.30697600046793616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,7168,0.13062933286031086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,7168,0.1436309337615967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,6144,0.22445333798726402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,6144,0.12479360103607177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,5120,0.186738125483195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,7168,0.25964372952779136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,5120,0.10688107013702393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,6144,0.1127285321553548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,4096,0.1508863925933838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,4096,0.08827199935913085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,4096,0.07798186937967935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,3584,0.06936000188191732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,3072,0.11329066753387451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,3584,0.13228586514790852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,3584,0.08305919965108235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,3072,0.07100266615549723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,2560,0.09314666589101156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,2560,0.05129706859588623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,2560,0.06807573636372885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,2048,0.07664960225423177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,1536,0.05825493335723877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,2048,0.047430400053660074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,1536,0.04279359976450602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,1024,0.03974719842274983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,1536,0.03307733337084452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,1024,0.03331519961357117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,1024,0.023897600173950196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,768,0.0285045325756073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,512,0.023754666248957314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,512,0.014786133170127868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,256,0.019381332397460937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,256,0.010573866963386535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,128,0.007667199770609538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,128,0.015175466736157736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,128,0.007828266421953837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,64,0.004123733441034953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,64,0.01572266618410746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,32,0.004045866678158442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,32,0.015913599729537965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,5120,0.09651947021484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,3072,0.05988053480784098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,51200,2048,0.05158933401107788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,768,0.030664533376693726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,51200,768,0.019697066148122153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,512,0.021186133225758873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,51200,256,0.01213653286298116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,65536,0.3771765391031901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,65536,0.7390741348266602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,16384,0.18485439618428548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,16384,0.12392000357309978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,65536,0.3541066805521647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,51200,0.27807467778523765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,51200,0.30024213790893556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,16384,0.11081706682840983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,12288,0.08397760391235351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,12288,0.14028159777323407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,51200,0.579862403869629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,12288,0.08395199775695801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,10240,0.11610666910807292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,8192,0.09477226734161377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,10240,0.07695679664611817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,10240,0.07067626317342122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,8192,0.06099626620610556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,7168,0.08236800034840902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,7168,0.04394346475601196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,7168,0.0556661327679952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,6144,0.07234559853871664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,8192,0.05752426783243815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,5120,0.059879465897878015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,6144,0.0496128002802531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,4096,0.04864319960276286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,5120,0.04381440083185832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,3584,0.043085865179697674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,4096,0.038022398948669434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,3072,0.03743040164311727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,3584,0.02489173412322998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,3584,0.03517866532007853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,3072,0.03249920010566711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,2560,0.03168960014979045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,3072,0.02213653326034546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,2560,0.029366399844487506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,2048,0.025821866591771443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,2560,0.019627734025319418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,2048,0.026125866174697875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,1536,0.020320000251134236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,1024,0.014351999759674073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,1536,0.01337493360042572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,1536,0.023491199811299643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,768,0.0115146666765213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,1024,0.010461866855621338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,1024,0.020542933543523153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,512,0.008550399541854858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,768,0.0192522664864858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,512,0.017270400126775106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,256,0.005723733206590017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,256,0.015059199929237366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,256,0.005310933291912079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,128,0.003446399917205175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,128,0.014801067113876343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,128,0.004825599988301595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,64,0.0031456001102924346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,64,0.014750933647155762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,32,0.0031615999837716425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,32,0.014807466665903726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,6144,0.03855573336283366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,768,0.009080533186594646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,65536,0.291755739847819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,65536,0.5518581390380859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,51200,0.23243412971496583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,51200,0.22525439262390137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,65536,0.2896021207173666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,51200,0.43345279693603517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,16384,0.1434922695159912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,16384,0.07988693714141845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,16384,0.09881813526153564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,12288,0.12881813049316407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,12288,0.07045653661092123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,12288,0.05782613356908163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,10240,0.09159146944681804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,10240,0.05793173313140869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,10240,0.048286934693654374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,8192,0.0747925360997518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,7168,0.06581759850184123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,8192,0.04037439823150635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,7168,0.045125333468119304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,7168,0.035879464944203694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,6144,0.04091413418451945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,5120,0.033191466331481935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,4096,0.027536000808080035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,6144,0.031111466884613036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,5120,0.04693546692530314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,5120,0.036400000254313156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,4096,0.04070933262507121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,4096,0.03204586704572042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,3584,0.03299946586290996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,4096,0.02299840052922567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,2048,0.016203733285268147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,3584,0.030105600754419964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,3072,0.028569600979487103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,3072,0.027485867341359455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,2560,0.0247978667418162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,2560,0.02545173366864522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,512,0.007500799993673961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,2048,0.020056533813476562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,2048,0.02331839998563131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,2048,0.013607466220855713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,1536,0.015845333536465965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,1536,0.02172266642252604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,1536,0.011560533444086711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,1024,0.011687466502189636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,8192,0.049116798241933185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,1024,0.01953173279762268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,768,0.009715200463930766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,1024,0.009928533434867859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,768,0.018207999070485432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,512,0.007502933343251546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,6144,0.054039466381073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,512,0.0057781333724657696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,512,0.015481600165367126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,256,0.004075733323891958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,5120,0.027141332626342773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,256,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,256,0.005202133456865946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,128,0.0034122665723164878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,128,0.014878933628400167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,128,0.004811733464399974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,64,0.0032277333239714304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,32,0.003138133386770884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,64,0.014937600493431092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,3072,0.018504534165064493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,32,0.014837333559989929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,65536,0.24779200553894043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,51200,0.3690762519836426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,65536,0.47260481516520186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,16384,0.14278507232666016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,51200,0.19990506172180175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,51200,0.18981547355651857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,16384,0.07555413246154785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,12288,0.09810453255971273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,12288,0.05909226735432943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,12288,0.048180266221364336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,10240,0.07886079947153726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,10240,0.051142398516337076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,10240,0.0413045326868693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,8192,0.06383466720581055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,8192,0.043993600209554035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,7168,0.052951467037200925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,3584,0.020859734217325846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,7168,0.04040426810582479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,7168,0.03100373347600301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,2560,0.01611733337243398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,6144,0.0464789350827535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,6144,0.037367467085520426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,6144,0.026689066489537554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,5120,0.03879893223444621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,5120,0.033140265941619874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,5120,0.023011199633280435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,4096,0.031726932525634764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,4096,0.02937920093536377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,4096,0.02013333241144816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,3584,0.028056534131368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,3072,0.025433599948883057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,3584,0.02757546703020732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,3072,0.025663999716440837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,3072,0.016235733032226564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,2560,0.02106346686681112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,2560,0.014313600460688271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,2048,0.01735573410987854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,2560,0.023919999599456787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,1536,0.01369599997997284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,2048,0.012578133742014566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,2048,0.022191999355951945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,1024,0.010294399658838908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,1536,0.010777599612871806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,1024,0.008775466680526733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,1024,0.01850879987080892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,768,0.008424533406893413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,768,0.016797866423924765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,768,0.006996266543865204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,512,0.015390933553377787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,768,0.008200533191363017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,512,0.005677866439024607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,256,0.0036138666172822317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,256,0.015027200182278952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,128,0.0032490665713946023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,128,0.014760532975196838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,3584,0.018516266345977785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,64,0.002915200094381968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,65536,0.23637439409891764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,64,0.014777599771817525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,32,0.002974933385848999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,32,0.014600533246994018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,1536,0.02051946719487508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,512,0.00666133314371109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,65536,0.1989855925242106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,65536,0.1859338601430257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,256,0.005184000233809153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,16384,0.0658357342084249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,65536,0.37245651880900066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,16384,0.059918932120005285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,51200,0.29604905446370444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,16384,0.06164480050404867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,12288,0.07027093569437662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,51200,0.1592917283376058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,12288,0.049218134085337324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,12288,0.039698131879170734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,10240,0.05868586699167887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,10240,0.034333864847819015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,8192,0.04784746567408244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,8192,0.038838398456573484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,8192,0.02837973237037659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,8192,0.034332799911499026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,7168,0.042242133617401124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,7168,0.03493333260218302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,6144,0.0368778665860494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,7168,0.025320533911387128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,5120,0.031052800019582112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,6144,0.03219306667645772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,6144,0.02251946727434794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,5120,0.019684267044067384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,5120,0.029313067595163982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,4096,0.017077332735061644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,4096,0.02605866591135661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,3584,0.022744532426198324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,3584,0.0247381329536438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,3072,0.0200437327226003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,3072,0.02329813241958618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,3072,0.013974400361378989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,2560,0.0172106663386027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,2560,0.021874133745829263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,2560,0.012522666652997335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,16384,0.09250880082448323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,2048,0.014203733205795288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,2048,0.011134933431943257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,2048,0.02025173306465149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,1536,0.011517866452534994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,1536,0.019086933135986327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,1024,0.008678399523099263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,1024,0.0172106663386027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,10240,0.043612798055013016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,768,0.007271466652552287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,128,0.004810666541258494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,768,0.0061951999862988796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,512,0.005715199808279673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,512,0.015398400028546652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,4096,0.025465599695841473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,256,0.0034783999125162757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,256,0.014787200093269347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,256,0.005002666513125102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,128,0.0031776001056035364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,128,0.014497066537539164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,128,0.004683733483155568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,64,0.002889599899450938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,64,0.014521599809328715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,32,0.0028736000259717304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,32,0.014498133460680643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,1536,0.009689600268999735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,65536,0.32395413716634114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,51200,0.14563093185424805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,768,0.015213867028554281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,65536,0.18036692937215168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,16384,0.08825813134511312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,51200,0.14524586995442706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,16384,0.05562026500701904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,51200,0.12879467010498047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,16384,0.04594133297602336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,12288,0.06495999892552694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,12288,0.046277332305908206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,12288,0.03610026836395264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,10240,0.05429759820302328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,10240,0.041179732481638594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,8192,0.04419306516647339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,10240,0.03091199994087219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,8192,0.0352789322535197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,8192,0.026260266701380413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,7168,0.03301653265953064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,3584,0.015476266543070475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,6144,0.035558398564656576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,6144,0.0302784005800883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,6144,0.02063680092493693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,5120,0.030343466997146608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,5120,0.027802666028340656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,5120,0.018262400229771932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,4096,0.023586134115854897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,4096,0.02500266631444295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,4096,0.015476266543070475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,3584,0.02058560053507487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,3584,0.02378666599591573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,3584,0.014166399836540222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,3072,0.01853013237317403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,3072,0.0225600004196167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,3072,0.0125408003727595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,2560,0.015640532970428465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,2560,0.021384533246358237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,2048,0.013174399733543396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,2560,0.011612799763679505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,2048,0.010212266445159912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,1024,0.007393066585063934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,2048,0.01990506649017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,1536,0.010903466741243999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,1536,0.00916373332341512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,1024,0.008025600016117096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,51200,0.25619093577067054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,1024,0.016872533162434897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,768,0.0068341334660847975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,1024,0.00714026689529419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,512,0.005499733487764994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,768,0.015460266669591268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,512,0.004164266586303711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,768,0.006178133189678192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,512,0.005606399973233541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,512,0.01513920029004415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,256,0.005085866649945577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,256,0.014868266383806863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,7168,0.03947519858678182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,128,0.0030623999734719592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,64,0.002919466545184453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,128,0.014648532867431641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,7168,0.02328959902127584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,128,0.0048320000370343525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,32,0.002919466545184453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,64,0.01442346672217051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,32,0.014517333110173544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,65536,0.15480213165283202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,65536,0.14211519559224445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,65536,0.27908906936645506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,16384,0.08734827041625977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,51200,0.12445653279622396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,51200,0.11308053334554036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,16384,0.05213866631189982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,12288,0.05679893493652344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,12288,0.04285973310470581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,1536,0.018642133474349974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,12288,0.03189333279927571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,10240,0.04877013365427653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,10240,0.03806399901707967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,10240,0.027459200223286944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,8192,0.038624000549316403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,8192,0.03368639945983887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,7168,0.033786666393280027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,8192,0.022729599475860597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,6144,0.03028586705525716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,7168,0.02076373298962911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,7168,0.031139200925827025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,256,0.0034858666360378264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,65536,0.16247572898864746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,5120,0.025914667050043742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,6144,0.01866133411725362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,6144,0.028908799091974895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,4096,0.021569067239761354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,4096,0.024283732970555624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,3584,0.019206400712331137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,3584,0.022977066040039063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,3584,0.012904533743858337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,3072,0.016736000776290894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,3072,0.021846399704615275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,3072,0.011603200435638427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,2560,0.014647466937700906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,2560,0.02062506675720215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,2560,0.011030399799346923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,2048,0.012265599767367045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,2048,0.0191648006439209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,2048,0.009484799702962239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,1536,0.0100000003973643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,1536,0.008691199620564779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,51200,0.22058347066243492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,1536,0.018295466899871826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,1024,0.007449600100517273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,1024,0.0063498665889104204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,768,0.0063285330931345625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,768,0.005915733178456625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,1024,0.015026133259137472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,512,0.004004266609748204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,512,0.0053162669142087305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,256,0.003332266708215078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,16384,0.03994986613591512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,256,0.014860799908638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,256,0.005006933212280273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,128,0.003032533327738444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,64,0.0028181334336598715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,128,0.004670933385690053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,128,0.014575999975204468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,32,0.0028917332490285236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,64,0.014682666460673014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,32,0.014598400394121806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,65536,0.13506453831990559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,65536,0.12302719751993815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,65536,0.23710293769836427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,5120,0.02659733295440674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,4096,0.013719466328620911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,16384,0.062414932250976565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,16384,0.0445248007774353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,51200,0.09785493214925131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,51200,0.10658559799194336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,16384,0.03545066515604655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,51200,0.18697813351949055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,12288,0.048344532648722335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,10240,0.041441067059834795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,12288,0.028251733382542926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,12288,0.03747306664784749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,8192,0.03326293428738912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,8192,0.02951573332150777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,8192,0.02025493383407593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,7168,0.02830186684926351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,7168,0.02764586607615153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,7168,0.01858133276303609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,6144,0.024626133839289348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,6144,0.02584853370984395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,768,0.015145599842071533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,5120,0.021078399817148843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,512,0.015079466501871744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,5120,0.014600533246994018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,5120,0.024089600642522177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,4096,0.01739413340886434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,4096,0.022132267554601036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,4096,0.012565333644549051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,3584,0.012087466319402058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,3584,0.02129813234011332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,3072,0.013916800419489542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,3072,0.020310399929682414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,3072,0.010803199807802836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,2560,0.019603200753529868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,2560,0.010294399658838908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,2048,0.010284800330797832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,2048,0.01875093380610148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,2048,0.008819199601809184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,1536,0.008658132950464885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,1536,0.01700586676597595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,1536,0.007739733159542084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,1024,0.006826666494210561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,1024,0.015372799833615622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,10240,0.03649919827779134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,768,0.005173333485921224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,768,0.014968533317248026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,768,0.005479466418425242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,512,0.0037151999771595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,512,0.014860799908638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,512,0.0051018665234247845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,256,0.003257599969704946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,3584,0.0156768004099528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,256,0.014644267161687216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,128,0.0029728000362714132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,128,0.01437333325544993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,2560,0.012142933408419291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,64,0.002868266652027766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,64,0.014567466576894126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,32,0.002855466554562251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,32,0.014548266927401224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,65536,0.18547093073527018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,5120,0.016156799594561257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,65536,0.10776746273040771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,65536,0.10599359671274822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,51200,0.14694827397664387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,10240,0.02418773372968038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,16384,0.04940906763076782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,51200,0.08472853501637777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,16384,0.03081066608428955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,16384,0.03833920160929362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,12288,0.03873279889424642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,6144,0.016702934106191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,12288,0.024871466557184856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,10240,0.03286400039990743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,12288,0.032953600088755294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,10240,0.030082132418950396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,8192,0.027015467484792072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,10240,0.021486934026082358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,8192,0.026923733949661254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,7168,0.024378667275110878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,8192,0.018285866578420004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,7168,0.025170133511225386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,7168,0.016622933745384216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,6144,0.02172586719195048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,6144,0.023704532782236734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,5120,0.018823466698328652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,6144,0.014973866939544677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,5120,0.02224853237469991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,4096,0.015625600020090738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,3584,0.01321386694908142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,4096,0.011560533444086711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,3584,0.01111893355846405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,3584,0.020081067085266115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,3072,0.011783466736475626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,3072,0.0100000003973643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,3072,0.019556266069412232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,2560,0.010365866621335347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,2560,0.009628799557685853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,2560,0.01884053349494934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,2048,0.008958933750788371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,2048,0.017798399925231932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,1536,0.0076682666937510175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,1536,0.015661866466204325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,1536,0.006982400019963582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,1024,0.00580266664425532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,51200,0.08727253278096517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,1024,0.0062282666563987735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,1024,0.005981866518656413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,768,0.003944533318281174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,768,0.015097600221633912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,768,0.0056309332450230915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,512,0.00359253336985906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,512,0.014879999558130899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,256,0.00505920002857844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,256,0.003173333406448364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,4096,0.0209279994169871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,256,0.014524799585342408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,128,0.002951466788848241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,128,0.004738133152325948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,128,0.014330666263898215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,128,0.004584533472855886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,64,0.014491732915242514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,32,0.0026943999032179515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,32,0.014457600315411887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,65536,0.16948800086975097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,1024,0.01542080044746399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,65536,0.09814613660176595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,512,0.005121066669623057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,65536,0.10223999818166096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,51200,0.1302016019821167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,51200,0.08138453165690104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,51200,0.08138773441314698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,16384,0.04723093509674072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,16384,0.035442133744557694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,12288,0.03513706525166829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,12288,0.030646399656931562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,12288,0.02291626731554667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,10240,0.0295199990272522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,256,0.004932266473770141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,10240,0.028182399272918702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,8192,0.024321067333221435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,64,0.00277866671482722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,8192,0.01763199965159098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,7168,0.02213653326034546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,7168,0.02404906749725342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,7168,0.016150400042533875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,6144,0.019399466117223103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,6144,0.02285760045051575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,6144,0.014619732896486918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,5120,0.017089066902796428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,5120,0.013240533073743186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,5120,0.021528534094492593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,4096,0.013330133756001792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,4096,0.020205867290496827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,4096,0.011467732985814412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,3584,0.012075733145078022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,2048,0.008217599987983704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,3584,0.019700266917546592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,3072,0.010819199681282043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,3072,0.01009173293908437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,3072,0.01908373236656189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,2560,0.009564800063769023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,2560,0.01843520005544027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,2560,0.009245866537094116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,2048,0.0083146666487058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,2048,0.01700586676597595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,1536,0.00713919997215271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,1536,0.015465600291887918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,1536,0.006923733154932658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,16384,0.030065067609151203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,1024,0.004882133503754934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,1024,0.006071466704209646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,1024,0.015294933319091797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,768,0.0039018665750821433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,768,0.005729066828886667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,10240,0.02101759910583496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,512,0.003491200009981791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,512,0.005285333096981049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,256,0.00308693324526151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,8192,0.02504853407541911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,5120,0.013339733084042868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,256,0.005005866785844167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,128,0.0029365333418051405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,256,0.014469333489735923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,128,0.00477866679430008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,64,0.002696533252795537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,128,0.01448319951693217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,32,0.0026943999032179515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,64,0.014500266313552857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,32,0.014360533157984415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,65536,0.08628479639689127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,51200,0.11325866381327312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,3584,0.010925867160161336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,51200,0.07029226620992025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,51200,0.07100906372070312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,16384,0.03928106625874837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,16384,0.03290773431460063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,12288,0.028386133909225463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,12288,0.02249600092569987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,10240,0.026229333877563477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,10240,0.026126933097839356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,10240,0.019504000743230186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,8192,0.0217631995677948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,8192,0.02445013324419657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,768,0.015326933066050211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,7168,0.019621332486470543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,512,0.01495573321978251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,7168,0.02344640096028646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,7168,0.015345066785812378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,6144,0.017271467049916587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,6144,0.022231467564900718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,6144,0.013875200351079305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,5120,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,5120,0.020875734090805054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,65536,0.09437759717305502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,65536,0.14248426755269367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,4096,0.01373546620210012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,4096,0.019579732418060304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,4096,0.010878933469454448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,3584,0.012168533603350322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,3584,0.018967467546463012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,3072,0.011173333724339802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,12288,0.031092266241709392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,3072,0.018422400951385497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,2560,0.008875733613967896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,2560,0.016816000143686928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,2048,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,2048,0.007838933169841767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,2048,0.016130133469899496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,2048,0.0075082664688428235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,1536,0.006702933212121327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,1536,0.01569066643714905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,1536,0.006725333134333293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,1024,0.004244266450405121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,1024,0.015321600437164306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,1024,0.005939200023810069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,768,0.003949866692225138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,768,0.015076266725858054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,768,0.005662933488686879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,512,0.0035264000296592714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,512,0.014923733472824097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,512,0.0052149335543314615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,256,0.0032149332265059153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,256,0.014595199624697366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,256,0.004975999891757965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,128,0.0028704000016053517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,16384,0.026257065931955974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,128,0.014459733168284097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,128,0.0047872001926104225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,64,0.014338133732477823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,32,0.002765866617361705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,32,0.014289066195487976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,65536,0.11907306512196858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,8192,0.016569599509239197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,65536,0.07433493137359619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,51200,0.09468586444854736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,65536,0.0929802656173706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,51200,0.06010666688283285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,16384,0.03270080089569092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,5120,0.012433066964149475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,16384,0.027533866961797077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,16384,0.03025599916776021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,12288,0.026195200284322102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,12288,0.026147200663884478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,12288,0.022016000747680665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,10240,0.022472532590230306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,10240,0.024652800957361855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,10240,0.01784106691678365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,8192,0.01889386574427287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,3584,0.010696533322334289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,8192,0.016798933347066246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,64,0.002690133452415466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,3072,0.009542399644851684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,7168,0.01718399922053019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,7168,0.015101866920789084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,6144,0.015511467059453329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,7168,0.021643733978271483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,5120,0.013557333747545877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,6144,0.02099626660346985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,6144,0.013755733768145243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,5120,0.020241065820058187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,4096,0.01913599967956543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,3584,0.010818133751551311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,3584,0.010397866368293762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,2560,0.008444799979527792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,3072,0.01004266639550527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,3072,0.016663466890652977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,2560,0.00823466678460439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,2560,0.01605226695537567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,2560,0.00806933343410492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,2048,0.007068799932797749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,2048,0.015863466262817382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,2048,0.007191466788450877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,1536,0.005749333401521047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,1536,0.01565226713816325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,1536,0.006542933483918508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,1024,0.004149333387613296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,1024,0.015335466464360556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,1024,0.005946666498978933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,768,0.003841066608826319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,768,0.015058133006095886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,8192,0.022486400604248048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,512,0.0034527999659379324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,512,0.01482133368651072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,4096,0.011622400085131327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,512,0.005077333251635233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,256,0.0031167998909950255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,4096,0.010838400324185688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,256,0.004940799872080485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,128,0.002883200099070867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,256,0.014686933159828186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,64,0.0026933332284291584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,128,0.004636799792448679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,128,0.014634666840235391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,32,0.002796799937884013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,64,0.014589866995811463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,32,0.014618666966756186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,65536,0.09612053235371908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,65536,0.06107626756032308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,51200,0.07463040351867675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,51200,0.05114986499150594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,51200,0.06815679868062338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,16384,0.026971733570098876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,16384,0.026819199323654175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,51200,0.07433066368103028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,12288,0.02153600056966146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,12288,0.023652267456054688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,12288,0.020364799102147422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,10240,0.018785067399342856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,10240,0.022502400477727256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,3584,0.018592000007629395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,10240,0.017914666732152303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,8192,0.01574399967988332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,8192,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,8192,0.021280000607172646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,7168,0.014341333508491516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,7168,0.020466132958730062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,5120,0.01243839959303538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,7168,0.014254933595657349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,6144,0.012831999858220419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,6144,0.012872533003489176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,5120,0.011689600348472596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,6144,0.019755733013153077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,5120,0.011738666892051696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,4096,0.010384000341097514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,5120,0.01953493356704712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,3072,0.009092266360918682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,4096,0.01776640017827352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,3584,0.016772266228993735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,3072,0.008937600255012512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,3072,0.016291200121243795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,768,0.005625600119431814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,2560,0.007602133353551229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,2560,0.015973333517710367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,2560,0.007864533364772797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,2048,0.006537599861621857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,2048,0.015793066223462424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,2048,0.007084799806276958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,1536,0.004870399832725525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,3584,0.00969599982102712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,65536,0.08354559739430746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,1536,0.0063391998410224915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,1024,0.003974399964014689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,1536,0.0155349334081014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,1024,0.005639466643333435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,1024,0.014972800016403198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,768,0.014973866939544677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,512,0.0034186666210492453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,512,0.01478506624698639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,512,0.005066666503747304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,256,0.0030005333324273427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,256,0.014501333236694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,256,0.004804266492525736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,128,0.002820266783237457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,128,0.014512000481287637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,64,0.00264533335963885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,64,0.014409599701563516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,32,0.0026634665826956432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,32,0.014335999886194864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,16384,0.02558186650276184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,65536,0.0734773317972819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,65536,0.050266667207082116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,65536,0.08289066950480142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,51200,0.058721065521240234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,51200,0.04322773218154907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,51200,0.06686186790466309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,16384,0.02182506720225016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,16384,0.024247467517852783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,16384,0.024465066194534302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,12288,0.017589332660039265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,12288,0.021959465742111207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,4096,0.010098133484522502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,10240,0.015505066514015198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,10240,0.01731200019518534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,768,0.0036757332583268487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,8192,0.013187199831008911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,8192,0.014949333667755128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,7168,0.012291199962298075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,7168,0.019375999768575035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,7168,0.013799466689427695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,3584,0.00981119970480601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,6144,0.011085866888364156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,128,0.004673066735267639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,6144,0.012492799758911132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,5120,0.009975467125574749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,3072,0.008441600203514098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,5120,0.011230933666229247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,5120,0.0186463991800944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,4096,0.008973866701126099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,4096,0.009386666615804036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,3584,0.008524800340334576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,4096,0.017037866512934367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,3072,0.007979733248551685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,3584,0.016722132762273155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,3072,0.016565333803494772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,3072,0.008108800152937572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,2560,0.007133866846561432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,2560,0.016122666994730632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,2560,0.0075765331586201985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,2048,0.005353599786758423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,2048,0.01593386630217234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,2048,0.006751999755700429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,1536,0.0047882666190465295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,1536,0.01572053333123525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,10240,0.020858667294184365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,1536,0.0063285330931345625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,1024,0.003987200061480204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,8192,0.019747199614842732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,1024,0.015370666980743408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,768,0.003714133302370707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,768,0.00537066658337911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,768,0.01506239970525106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,512,0.003307733436425527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,512,0.014847999811172486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,512,0.004986666639645894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,256,0.0030720000465710956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,256,0.014625066518783569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,256,0.00481279989083608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,768,0.0054058666030565895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,128,0.002775466690460841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,64,0.0026869334280490874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,128,0.0045952002207438145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,128,0.01456000010172526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,32,0.002682666728893916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,64,0.014547200004259745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,32,0.014430933197339377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,65536,0.049464531739552814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,3584,0.008955732981363932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,65536,0.03866666555404663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,51200,0.03946773211161296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,16384,0.015863466262817382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,51200,0.06613119840621948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,16384,0.021367466449737547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,16384,0.024502400557200113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,12288,0.013265066345532737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,12288,0.019657599925994872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,12288,0.01958720088005066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,10240,0.011955199639002483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,10240,0.01954986651738485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,10240,0.017196800311406454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,8192,0.010392533739407857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,8192,0.01831573247909546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,7168,0.009910399715105694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,7168,0.017579734325408936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,7168,0.01323946714401245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,6144,0.008745599786440532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,6144,0.01723946730295817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,6144,0.011586133639017742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,5120,0.008282666901747386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,12288,0.020053333044052123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,5120,0.017867734034856163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,4096,0.007582933207352956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,6144,0.0186954657236735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,4096,0.017131733894348144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,3584,0.006945066650708516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,4096,0.009301333626111349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,3584,0.008802133798599242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,3584,0.01694613297780355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,3072,0.00626453310251236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,3072,0.01649386684099833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,2560,0.006098133325576782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,51200,0.033574398358662924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,2560,0.016158933440844216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,2048,0.005435733497142792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,2560,0.007521066566308339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,2048,0.006785066425800323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,2048,0.015876266360282897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,1536,0.004741333425045013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,1536,0.01570026675860087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,1536,0.006343466540177663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,1024,0.004016000032424927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,1024,0.015066666404406228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,1024,0.005648000041643778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,768,0.003623466690381368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,8192,0.014728533228238425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,768,0.015209600329399109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,768,0.0052490666508674625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,512,0.0033791999022165934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,256,0.0030069333811601003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,512,0.005058133105436961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,256,0.014678399761517844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,128,0.002799999962250392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,128,0.0045952002207438145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,128,0.01482133368651072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,64,0.0029343999922275543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,32,0.0028927999238173166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,64,0.01478506624698639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,32,0.01458346645037333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,5120,0.010498133301734925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,65536,0.03337706724802653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,51200,0.031446399291356404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,65536,0.08399786949157714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,1024,0.0056991999348004665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,51200,0.0294922669728597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,51200,0.06649066607157389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,16384,0.020230400562286376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,16384,0.024661332368850708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,65536,0.083024001121521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,12288,0.01127893328666687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,12288,0.019713066021601357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,12288,0.018768000602722167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,10240,0.010311466455459595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,10240,0.018252799908320107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,10240,0.017179733514785765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,8192,0.009284266829490661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,8192,0.017199999094009398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,8192,0.014439466595649719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,7168,0.008532266815503438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,7168,0.017550933361053466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,6144,0.007810133198897044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,512,0.015033599734306336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,6144,0.017167999347050985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,5120,0.007276799778143566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,5120,0.017838933070500693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,5120,0.010418132940928141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,4096,0.006178133189678192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,65536,0.038696531454722086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,4096,0.017094399531682333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,4096,0.009146666526794434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,3584,0.0064533332983652755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,3584,0.008840533097585042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,3584,0.016778665781021117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,3072,0.0060362666845321655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,3072,0.01646506687005361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,3072,0.007957333326339721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,2560,0.005987200140953064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,2560,0.016122666994730632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,2560,0.007500799993673961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,16384,0.013265066345532737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,2048,0.005386666456858317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,1536,0.004700799783070883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,2048,0.006748799979686737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,2048,0.01588266690572103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,1024,0.003953066716591517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,1536,0.006390400230884552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,1536,0.015681067109107973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,768,0.003722666700681051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,1024,0.0055871998270352686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,1024,0.01532373329003652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,512,0.0033098667860031127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,768,0.015334399541219077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,512,0.0050346667567888895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,512,0.015042133132616677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,256,0.0032479998966058097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,256,0.014748799800872802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,256,0.004823466638724009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,128,0.0029663999875386557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,128,0.0047189335028330484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,128,0.01453013320763906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,64,0.003036800026893616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,32,0.0027562665442625684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,64,0.014628266294797262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,32,0.014634666840235391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,6144,0.011554132898648579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,65536,0.027702399094899494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,65536,0.08298666477203369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,51200,0.024822400013605753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,51200,0.06592746575673422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,256,0.004752000172932943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,16384,0.010363733768463135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,16384,0.01928960084915161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,16384,0.024100265900293984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,12288,0.009048533439636231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,12288,0.017139200369517008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,12288,0.01917120019594828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,10240,0.008505599697430928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,10240,0.017484800020853678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,10240,0.017035732666651406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,8192,0.007707733412583668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,8192,0.01730666756629944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,8192,0.01400213340918223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,7168,0.0066549330949783325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,7168,0.017490132649739584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,7168,0.012853333353996277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,6144,0.006074666480223338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,6144,0.01738133430480957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,768,0.005367466807365417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,6144,0.011620266238848369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,5120,0.006265600025653839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,5120,0.01037440001964569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,4096,0.005780266722043356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,5120,0.017825067043304443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,3072,0.00800960014263789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,3584,0.0063391998410224915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,4096,0.017268266280492148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,3072,0.005974400043487549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,3584,0.008734933535257975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,3072,0.016502400239308677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,3072,0.00795413355032603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,2560,0.007575466732184092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,2560,0.01635840038458506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,2048,0.005436799923578898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,2048,0.01602133313814799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,2048,0.0066442668437957765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,65536,0.027185066541035967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,1536,0.004814933240413666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,1536,0.006381866832574208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,1536,0.01565439999103546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,7168,0.013290666540463767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,1024,0.0055754666527112326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,1024,0.015496533115704855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,768,0.0037823999921480812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,768,0.005330133438110352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,768,0.015147733688354491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,512,0.003521066655715307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,512,0.004911999901135763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,512,0.01483519971370697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,256,0.0031082667410373688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,256,0.004819199939568837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,128,0.0028949332733949023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,128,0.014549332857131957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,128,0.004584533472855886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,64,0.0027594665686289472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,64,0.014551466703414917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,32,0.002780800064404805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,32,0.01458026667435964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,65536,0.01685439944267273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,65536,0.02265920042991638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,4096,0.009365333120028178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,3584,0.01678933302561442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,51200,0.014339199662208557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,2560,0.0061749334136645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,65536,0.08332800070444743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,16384,0.0075328002373377485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,51200,0.02058560053507487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,51200,0.06566933393478394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,16384,0.017534933487574258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,12288,0.0063360000650088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,16384,0.023681066433588662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,10240,0.006138666470845541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,12288,0.018900267283121743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,8192,0.006196266909440359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,10240,0.01764799952507019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,10240,0.016350932916005454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,7168,0.00603413333495458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,8192,0.013988266388575235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,8192,0.017172267039616905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,6144,0.005910400052865346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,51200,0.022336000204086305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,7168,0.017414400974909462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,5120,0.006180266539255777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,6144,0.011544533570607503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,6144,0.017193599541982015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,4096,0.005874133110046387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,5120,0.010297600428263347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,5120,0.017783466974894205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,3584,0.00626453310251236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,4096,0.017113600174585977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,4096,0.009061333537101746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,3584,0.0088128000497818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,3072,0.005899733304977417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,3584,0.016760534048080443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,3072,0.016456533471743265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,3072,0.007857066889603932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,256,0.01470186710357666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,2560,0.016062933206558227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,2048,0.015773866573969522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,2048,0.0067210664351781205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,1536,0.004699733356634776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,1536,0.015457066893577575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,1536,0.006340266764163971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,1024,0.003950933367013932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,1024,0.015236266454060874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,1024,0.005502933263778686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,768,0.0035829332967599234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,768,0.015030399958292643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,768,0.0052597333987553915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,512,0.0032874666154384612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,12288,0.017080533504486083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,512,0.014858667055765787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,512,0.005048533280690512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,256,0.00296426663796107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,256,0.004669866462548574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,256,0.014504533012708029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,1024,0.004074666649103165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,7168,0.012865066528320312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,128,0.0028319999575614927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,64,0.002631466587384542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,128,0.014390400052070618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,128,0.004554666578769684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,32,0.0026176000634829206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,65536,0.011409067114194234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,32,0.014493866761525472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,65536,0.019364267587661743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,51200,0.010140800476074218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,16384,0.005962666869163513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,51200,0.01887786587079366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,51200,0.06520533164342245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,16384,0.02363199989000956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,16384,0.01740586757659912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,12288,0.005751466751098633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,10240,0.005914666752020518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,2048,0.005331199864546458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,12288,0.01877760092417399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,12288,0.016976000865300496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,10240,0.017303466796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,8192,0.005905066430568695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,10240,0.016323199868202208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,7168,0.005758933226267497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,8192,0.014051199952761332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,8192,0.017169066270192466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,7168,0.01279146671295166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,6144,0.005612800021966299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,7168,0.017454934120178223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,5120,0.005940266450246175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,6144,0.011480533083279927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,5120,0.010333866874376933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,5120,0.017589332660039265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,4096,0.00553173323472341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,4096,0.016990933815638223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,4096,0.009196799993515015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,3584,0.006016000111897787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,3584,0.016504533092180886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,3584,0.008717866738637288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,3072,0.0057087997595469155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,3072,0.007993599772453308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,3072,0.016235733032226564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,2560,0.005755733450253805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,2560,0.01591253379980723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,2560,0.0074080000321070354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,2048,0.00514986664056778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,2048,0.01574186682701111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,2048,0.0066890666882197065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,1536,0.00446720023949941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,1536,0.0063498665889104204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,1536,0.015682133038838704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,1024,0.0038880000511805216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,1024,0.015220266580581666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,1024,0.005538133283456167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,768,0.003606399893760681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,768,0.00525439977645874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,768,0.015003732840220132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,512,0.003399466723203659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,512,0.004884266853332519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,512,0.014740266402562461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,256,0.0029866665601730345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,256,0.014628266294797262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,128,0.0027797333896160126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,128,0.014477866888046264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,128,0.004491733511288961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,64,0.002682666728893916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,6144,0.017092265685399375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,32,0.002666666607062022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,2560,0.007500799993673961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,65536,0.00860479970773061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,65536,0.018116267522176106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,51200,0.007515733440717061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,51200,0.01816640098889669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,16384,0.0057322666049003605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,16384,0.017044266064961754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,12288,0.0056426664193471275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,12288,0.016951467593510947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,10240,0.005862399935722351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,10240,0.017027199268341064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,8192,0.005688533186912537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,2560,0.00600853314002355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,8192,0.016720000902811685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,7168,0.005832533538341522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,256,0.004753066599369049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,6144,0.005689600110054016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,7168,0.017090133825937905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,5120,0.006075733403364817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,6144,0.01691306630770365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,4096,0.005731200178464254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,5120,0.01725546717643738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,4096,0.016810667514801026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,3584,0.006095999975999197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,65536,0.082204802831014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,3072,0.005671466886997223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,2560,0.005570133527119955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,3584,0.016302933295567833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,3072,0.016029866536458333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,2048,0.0050005331635475155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,2560,0.015920000274976094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,1536,0.004386133452256521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,2048,0.015459199746449789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,1536,0.015228799978892007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,1024,0.0038399999340375268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,768,0.0034143999218940735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,512,0.003151999910672506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,1024,0.015049599607785544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,32,0.014416000247001648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,256,0.002930133293072383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,512,0.014708266655604044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,128,0.002792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,256,0.014552533626556396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,64,0.0025920001169045764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,128,0.014602667093276978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,32,0.0026869334280490874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,64,0.01437333325544993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,65536,0.007666133344173431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,32,0.014411733547846476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,51200,0.007658666869004567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,65536,0.017757866779963175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,16384,0.005692799886067709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,51200,0.017767467101415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,12288,0.0057205334305763245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,10240,0.005771733323733012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,12288,0.016964266697565712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,8192,0.00573333352804184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,8192,0.016785067319869996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,7168,0.0055744002262751256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,10240,0.017361066738764443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,6144,0.005479466418425242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,7168,0.01727893352508545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,64,0.014379733800888061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,5120,0.005845333139101664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,6144,0.01696959932645162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,4096,0.0055071999629338585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,3584,0.005905066430568695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,5120,0.017436800400416057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,3072,0.005633066594600678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,3584,0.016300800442695617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,2560,0.0055637334783871974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,3072,0.016233600179354348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,2048,0.004965333143870035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,2560,0.015727999806404113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,1536,0.0043935999274253845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,2048,0.015597866972287497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,1536,0.015152000387509666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,1024,0.00383786658445994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,768,0.0034656000634034475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,1024,0.0150026669104894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,512,0.003197866678237915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,768,0.014537599682807923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,512,0.014724266529083253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,128,0.0027530667682488757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,256,0.014292266964912415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,64,0.002587733417749405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,128,0.014296533664067588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,64,0.01442346672217051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,32,0.0026591998835404714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,64,0.014178133010864258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,32,0.014427733421325684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,768,0.015041066209475198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,16384,0.017566933234532674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,4096,0.016591999928156534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,16384,0.7452309290568034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,16384,0.39282026290893557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,16384,0.35521707534790037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,12288,0.5641269048055013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,12288,0.2971872011820475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,12288,0.26929492950439454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,51200,1.0939114888509116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,51200,1.1782805124918618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,10240,0.24938987096150717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,10240,0.22433066368103027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,256,0.002889599899450938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,10240,0.4712575912475586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,8192,0.23876266479492186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,8192,0.18101973533630372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,8192,0.4102304140726726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,7168,0.18589760462443033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,7168,0.3287754694620768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,7168,0.1785216013590495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,51200,2.3358133951822917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,6144,0.2840970675150553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,6144,0.15509653091430664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,5120,0.11472000281016033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,5120,0.13204480012257894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,5120,0.2363488038380941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,4096,0.09289173285166422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,4096,0.19230400721232097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,4096,0.12493120034535725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,3584,0.16930774052937825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,3584,0.09729066689809164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,3584,0.08205973307291667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,3072,0.08597226937611899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,3072,0.15903253555297853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,2560,0.0738165299097697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,2560,0.0599402666091919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,3072,0.071234130859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,2048,0.0958346684773763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,2048,0.049192531903584795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,2048,0.06222079992294312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,1536,0.07363093694051107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,1536,0.05079573392868042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,1024,0.038575998942057294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,1024,0.04987093210220337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,1536,0.0382805347442627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,1024,0.02770666678746541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,512,0.02656746705373128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,768,0.02195626695950826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,512,0.01667520006497701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,768,0.03294826745986938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,512,0.02674773335456848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,256,0.014843733112017313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,128,0.00904960036277771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,256,0.012132267157236736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,256,0.02076266606648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,128,0.009082667032877604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,64,0.006100266675154368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,128,0.019530665874481202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,32,0.018653867642084758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,64,0.01869973341623942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,6144,0.13756160736083983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,2560,0.1413706620534261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,768,0.03896213372548421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,32,0.004321066538492838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,65536,1.1932596842447916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,16384,0.5947786966959636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,51200,0.9394720077514649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,16384,0.2917386690775553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,51200,0.9078144073486328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,51200,1.8372149149576824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,16384,0.3522282600402832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,65536,2.3412256876627606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,12288,0.23683519363403321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,10240,0.19982825915018718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,12288,0.44791040420532224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,12288,0.2166111946105957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,10240,0.21743146578470865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,8192,0.30609280268351235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,10240,0.4388714790344238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,8192,0.1645311991373698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,8192,0.1472607930501302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,7168,0.1428554693857829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,7168,0.15186986923217774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,6144,0.22399039268493653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,7168,0.2774335861206055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,6144,0.1313546657562256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,6144,0.14351253509521483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,5120,0.18628053665161132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,4096,0.07787306308746338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,5120,0.10661013126373291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,5120,0.11121280193328857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,3584,0.13350720405578614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,4096,0.08882773717244466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,4096,0.15085973739624023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,3584,0.07878826459248861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,3584,0.06806613604227701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,3072,0.05952320098876953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,3072,0.11316800117492676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,2560,0.09469760258992513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,3072,0.07056106726328532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,2560,0.05075519879659017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,2560,0.060540799299875894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,2048,0.05194453398386637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,1536,0.05791999896367391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,1536,0.042290135224660234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,1024,0.03978346586227417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,768,0.030642133951187134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,1024,0.026523733139038087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,512,0.02137813369433085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,768,0.028387200832366944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,512,0.023659733931223552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,256,0.011945600310961407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,256,0.01914560000101725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,256,0.010631466905275982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,128,0.007499733567237854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,128,0.015081600348154704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,65536,1.1407199859619142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,128,0.007709866762161255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,64,0.015661866466204325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,64,0.004005333284536997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,32,0.003988266736268997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,32,0.015545599659283958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,51200,2048,0.07716693083445231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,2048,0.041740798950195314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,51200,1024,0.0333621342976888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,768,0.018875734011332194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,512,0.014944000045458474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,65536,0.37715625762939453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,65536,0.7389386494954426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,16384,0.1845695972442627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,16384,0.10621226628621419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,51200,0.3006410598754883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,65536,0.35465173721313475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,16384,0.110698668162028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,51200,0.5790784200032552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,12288,0.13980159759521485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,12288,0.08380053043365479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,12288,0.07163626352945963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,10240,0.13484692573547363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,8192,0.06065066655476888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,8192,0.09422933260599772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,8192,0.05750720103581747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,10240,0.0705130656560262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,7168,0.08228800296783448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,7168,0.05498239994049072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,6144,0.07212373415629068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,7168,0.04373226563135783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,6144,0.04930026531219482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,5120,0.059831468264261875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,5120,0.033063467343648276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,4096,0.048631465435028075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,4096,0.03762773275375366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,4096,0.02735466758410136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,3584,0.042855465412139894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,3584,0.035070931911468504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,3072,0.03721813360850017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,51200,1536,0.0370250662167867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,2560,0.03165760040283203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,3072,0.03205759922663371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,2560,0.02929813265800476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,10240,0.07408640384674073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,2048,0.03004053235054016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,2048,0.026130133867263795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,5120,0.04367893139521281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,1536,0.02026559909184774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,1536,0.0233024001121521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,3584,0.0247488001982371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,1024,0.014456533392270408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,3072,0.021962666511535646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,1024,0.02032853364944458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,1024,0.010397866368293762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,768,0.011534933249155681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,768,0.018999467293421425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,768,0.009054932991663616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,512,0.008619733651479085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,512,0.016574933131535848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,512,0.007603199779987335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,256,0.005524266759554545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,256,0.005317333340644837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,256,0.014988799889882406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,128,0.003454933315515518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,128,0.014588800072669984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,128,0.004759466648101807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,64,0.0031306666632493338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,64,0.014755200346310934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,32,0.003147733211517334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,32,0.014838400483131408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,51200,0.27826452255249023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,65536,0.29103466669718425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,65536,0.5516586939493815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,6144,0.03854506810506185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,65536,0.27828585306803383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,51200,0.2321546713511149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,51200,0.4331818580627441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,16384,0.14246932665507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,51200,0.22510080337524413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,2560,0.019576533635457357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,16384,0.09242560068766276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,16384,0.07424533367156982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,12288,0.1287775993347168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,12288,0.06736640135447183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,10240,0.09120852947235107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,10240,0.04764159917831421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,10240,0.059579734007517496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,8192,0.050327467918396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,8192,0.07445333003997803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,7168,0.06566826502482095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,7168,0.04586133162180583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,7168,0.03588586648305257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,6144,0.059571198622385656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,6144,0.04045546849568685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,6144,0.031345067421595256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,5120,0.04668906529744466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,5120,0.02704746723175049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,5120,0.03666986624399821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,4096,0.03714453379313151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,4096,0.03244373401006063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,4096,0.022626133759816487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,3584,0.035819733142852785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,3584,0.030002133051554365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,3072,0.029045333464940388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,3584,0.020848000049591066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,3072,0.027852799495061236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,3072,0.018307199080785118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,2560,0.01628373364607493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,2048,0.01623466710249583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,2048,0.023553067445755006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,1536,0.016235733032226564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,2048,0.013547733426094055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,1536,0.011506133278210958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,1536,0.021693867444992066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,1024,0.011748266220092774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,1024,0.019409066438674925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,768,0.009704533219337463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,1536,0.013666133085886637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,768,0.018292266130447387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,768,0.008110933502515157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,512,0.007549866537253062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,512,0.01574186682701111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,256,0.004030933231115341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,256,0.01520853340625763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,12288,0.06702079772949218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,128,0.0034933333595593774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,128,0.014841600259145101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,128,0.004750933249791463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,64,0.003223466624816259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,64,0.014940800269444785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,32,0.0031328000128269195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,32,0.014939733346303306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,2560,0.0243285338083903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,2048,0.020097066958745323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,65536,0.24705492655436195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,65536,0.4714912096659343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,65536,0.23434665997823081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,51200,0.1896000067392985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,51200,0.19939732551574707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,51200,0.3680202802022298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,16384,0.12332373460133869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,16384,0.0809706687927246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,8192,0.04026559988657634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,16384,0.07450986703236898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,12288,0.09312746524810792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,12288,0.058369068304697666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,12288,0.04952106475830078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,10240,0.07842880090077718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,2560,0.02560960054397583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,10240,0.05111786524454752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,10240,0.04203199942906698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,8192,0.06373226642608643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,7168,0.05332693258921305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,7168,0.04038399855295817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,7168,0.03100373347600301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,6144,0.04654719829559326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,6144,0.036735999584198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,6144,0.02725333372751872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,5120,0.03902293443679809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,5120,0.033070933818817136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,5120,0.023572266101837158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,4096,0.031729066371917726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,4096,0.02929813265800476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,4096,0.020026665925979615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,1024,0.009223467111587525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,3584,0.02815679907798767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,3584,0.02736639976501465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,3584,0.018100267648696898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,3072,0.024569600820541382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,2560,0.02090880076090495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,3072,0.016217600305875143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,2560,0.014603733023007711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,2560,0.02373653252919515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,512,0.005746133128801982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,2048,0.01718613306681315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,2048,0.02193386753400167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,256,0.005161599814891815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,1536,0.010501333077748616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,1024,0.010337066650390626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,1024,0.018501333395640054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,1024,0.00869546631971995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,768,0.008564266562461852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,768,0.016039466857910155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,768,0.0070250665148099255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,512,0.006648533542950948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,512,0.015384533007939658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,512,0.0056981335083643595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,256,0.0036757332583268487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,256,0.015037866433461508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,3072,0.02551146745681763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,128,0.003219199925661087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,128,0.014500266313552857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,1536,0.013688533504803976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,64,0.003011200080315272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,1536,0.020397865772247316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,64,0.014666666587193808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,32,0.002890666574239731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,32,0.014769066373507181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,8192,0.04368319908777873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,65536,0.19763627052307128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,65536,0.37196693420410154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,8192,0.03502613306045532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,65536,0.18457066218058268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,51200,0.1645802656809489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,16384,0.09203519821166992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,51200,0.14473600387573243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,16384,0.06213866472244263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,16384,0.05199679931004843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,12288,0.07033812999725342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,12288,0.04919039805730184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,12288,0.04007680018742879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,10240,0.05856000185012818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,10240,0.043271466096242266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,8192,0.04789653221766154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,10240,0.03388693332672119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,8192,0.0387605349222819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,7168,0.042206935087839764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,7168,0.03481066624323527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,2048,0.012542933225631714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,8192,0.028207999467849732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,6144,0.03191786607106527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,6144,0.023085866371790567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,5120,0.031128533681233722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,5120,0.028961066404978437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,5120,0.0202239990234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,4096,0.025382399559020996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,4096,0.026088533798853557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,4096,0.017090133825937905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,3584,0.015379200379053751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,3584,0.024661332368850708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,3072,0.01995413303375244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,3072,0.02323413292566935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,3072,0.013670399785041809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,2560,0.017126399278640746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,2560,0.022024534145991006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,2560,0.01246506671110789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,2048,0.014257066448529563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,2048,0.020406399170557657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,2048,0.011128532886505126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,1536,0.011494400103886922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,1536,0.009644800424575805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,1536,0.019146666924158732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,51200,0.29395840962727865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,1024,0.008584533135096233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,256,0.005063466727733612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,1024,0.00763733337322871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,768,0.007222400108973186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,1024,0.01801813244819641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,768,0.015411200126012168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,512,0.005757866799831391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,512,0.015440000096956888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,512,0.005423999826113383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,256,0.003500800083080927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,256,0.005058133105436961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,256,0.01492800017197927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,128,0.0031338666876157125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,128,0.014631467064221701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,128,0.004648533463478088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,64,0.0028490667541821797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,32,0.002881066749493281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,6144,0.03708693186442057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,32,0.014725333452224732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,128,0.004704000055789947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,3584,0.02267520030339559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,65536,0.32485227584838866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,65536,0.16210133234659832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,51200,0.25505599975585935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,51200,0.1444053332010905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,16384,0.08770026365915934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,51200,0.12889493306477864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,16384,0.05528853336970011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,12288,0.06474773486455282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,16384,0.045754667123158774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,12288,0.045685335000356035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,12288,0.03572053511937459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,10240,0.05389973322550455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,10240,0.04080853462219238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,8192,0.044064001242319746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,10240,0.03049280047416687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,8192,0.035148799419403076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,64,0.014794666568438211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,7168,0.03933013280232747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,8192,0.025226666529973345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,7168,0.0253983994325002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,7168,0.02285333275794983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,6144,0.030433066685994464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,5120,0.029487999280293782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,5120,0.027738666534423827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,5120,0.017986132701237997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,4096,0.023124267657597862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,4096,0.025169066588083905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,4096,0.015229866902033488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,3584,0.02067306637763977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,3584,0.014077867070833841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,3584,0.023944532871246337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,3072,0.018336000045140584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,65536,0.17943894068400065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,3072,0.022734934091567995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,2560,0.015668267011642457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,2560,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,2560,0.01162559986114502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,2048,0.013138133287429809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,2048,0.010227200388908387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,2048,0.019960532585779824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,1536,0.010668800274531046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,1536,0.018872533241907755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,1024,0.008060800035794576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,1024,0.015666133165359496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,1024,0.0066655998428662615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,768,0.0067893331249554946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,768,0.015227733055750528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,768,0.006073600053787232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,512,0.004263466596603394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,768,0.005899733304977417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,512,0.005500799914201101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,512,0.015080533425013223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,256,0.003519999980926514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,6144,0.034622931480407716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,128,0.003101866692304611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,256,0.014935466647148132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,64,0.002867199977238973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,128,0.004727466901143392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,128,0.014549332857131957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,32,0.002919466545184453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,64,0.014754133423169455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,32,0.01456106702486674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,65536,0.14192105929056803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,65536,0.28137280146280924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,1536,0.009124267101287841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,51200,0.2201162656148275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,51200,0.12430826822916667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,16384,0.07398613293965658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,16384,0.05067413250605265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,16384,0.040277334054311116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,12288,0.05666559934616089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,51200,0.11303679943084717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,12288,0.0425866683324178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,12288,0.03170666694641113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,10240,0.04861973524093628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,10240,0.027322665850321455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,10240,0.03792426586151123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,7168,0.032680533329645794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,8192,0.03334933519363403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,8192,0.022721066077550253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,7168,0.03382506767908732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,7168,0.031137067079544067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,7168,0.0206112007300059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,6144,0.030179200569788615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,6144,0.028429865837097168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,6144,0.018529067436854042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,5120,0.025229867299397784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,5120,0.026437334219614667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,6144,0.020598399639129638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,4096,0.0210207998752594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,65536,0.15454400380452474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,4096,0.013965866963068643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,3584,0.020036266247431437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,3584,0.022714666525522866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,3072,0.016807466745376587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,3584,0.013058132926623025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,3072,0.011621333161989848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,3072,0.021757866938908896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,2560,0.014444800217946372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,3072,0.012611200412114462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,2560,0.020488532384236653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,2560,0.010824533303578694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,2048,0.012449066837628682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,1536,0.010018133123715718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,2048,0.009478400150934856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,2048,0.019282132387161255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,1536,0.0086325337489446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,1536,0.01830613414446513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,1024,0.007542400062084198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,1024,0.006234666705131531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,768,0.006364800035953522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,1024,0.015440000096956888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,512,0.004092800120512644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,768,0.015339733163515726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,512,0.005268266797065735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,512,0.015146666765213012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,256,0.0033621333539485933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,256,0.014879999558130899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,128,0.003058133274316788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,256,0.00525546669960022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,128,0.014717866977055868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,128,0.004706133405367533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,64,0.0028480000793933867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,32,0.0028970666229724885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,64,0.01451520025730133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,32,0.014617600043614707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,5120,0.016132266322771708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,65536,0.13479785919189452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,65536,0.23604373931884765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,65536,0.12231252988179524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,51200,0.10580373605092366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,51200,0.18621865908304852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,51200,0.09795626799265543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,16384,0.06289173364639282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,16384,0.044494934876759845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,12288,0.047942399978637695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,16384,0.03716053167978923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,12288,0.03713279962539673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,10240,0.041725865999857586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,12288,0.028124799331029255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,768,0.005881600081920624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,8192,0.03323413332303365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,8192,0.029577600955963134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,8192,0.020106667280197145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,7168,0.027853866418202717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,7168,0.027550933758417766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,8192,0.03978559970855713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,7168,0.018659200270970663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,6144,0.02476373314857483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,5120,0.021217066049575805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,6144,0.016708266735076905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,6144,0.025803732872009277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,5120,0.014546133081118264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,4096,0.017459199825922648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,4096,0.012719999750455221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,4096,0.022453333934148154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,3584,0.015564800302187601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,3584,0.02143893241882324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,3584,0.012000000476837159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,3072,0.013870933651924133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,3072,0.020457599560419717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,2560,0.01213653286298116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,4096,0.02427306572596232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,2560,0.010243200262387593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,2560,0.01962560017903646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,2048,0.010366933544476827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,2048,0.01851093371709188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,2048,0.00883840024471283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,1536,0.008701866865158081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,1536,0.016641066471735636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,1536,0.007772799829641979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,1024,0.006734933455785115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,1024,0.006274133423964183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,1024,0.015099733074506124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,768,0.005352533360322317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,768,0.015185067057609558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,512,0.003853866706291834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,10240,0.03332479993502299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,512,0.015122133493423461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,512,0.0053269331653912864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,256,0.0032586666444937387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,256,0.014739200472831726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,256,0.005016533533732096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,128,0.0029440000653266907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,128,0.014613333344459533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,128,0.0047882666190465295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,64,0.0028319999575614927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,32,0.0028309332827727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,64,0.01451520025730133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,5120,0.02401919960975647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,256,0.004986666639645894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,32,0.014587733149528503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,65536,0.10712532997131348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,65536,0.18573333422342936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,51200,0.1467082659403483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,3072,0.01079360047976176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,16384,0.049456000328063965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,16384,0.038424531618754074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,51200,0.08749547004699706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,51200,0.08454933166503906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,12288,0.03885120153427124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,12288,0.024762666225433348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,10240,0.03269333243370056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,12288,0.032959999640782674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,10240,0.02990399996439616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,8192,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,8192,0.01800000071525574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,8192,0.026796799898147584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,7168,0.024378667275110878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,7168,0.025191466013590496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,7168,0.016620799899101257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,6144,0.021706666549046835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,6144,0.02379093368848165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,6144,0.014814933141072592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,5120,0.018920532862345376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,5120,0.022333866357803343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,5120,0.013224533200263977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,4096,0.015980799992879234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,4096,0.011576533317565918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,4096,0.020966400702794395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,3584,0.013215999801953635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,3584,0.020066134134928384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,3584,0.011176533500353495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,3072,0.011879466970761617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,10240,0.024375466505686443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,3072,0.010155733426411946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,2560,0.010475732882817586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,2560,0.018708266814549766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,2560,0.009450667103131612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,2048,0.00906880001227061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,2048,0.017718400557835898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,1536,0.007595733304818471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,1536,0.015709867080052696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,1536,0.00660693347454071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,1024,0.006074666480223338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,1024,0.01520639955997467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,1024,0.005657599866390228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,768,0.003966933240493139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,768,0.01495573321978251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,10240,0.0214954674243927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,768,0.00537066658337911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,512,0.0035786665976047516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,512,0.014906666676203408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,512,0.00521919975678126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,256,0.003155199935038885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,256,0.014735999703407287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,128,0.002979200085004171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,128,0.014519466956456503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,128,0.004745600124200186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,64,0.002844800055027008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,768,0.005814399818579356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,64,0.014382933576901754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,32,0.014550399780273438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,3072,0.019530665874481202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,65536,0.16676586469014484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,65536,0.09833813508351644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,51200,0.12992853323618572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,65536,0.10326826572418213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,16384,0.0307093342145284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,51200,0.08159680366516113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,65536,0.1062282641728719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,16384,0.04666879971822103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,16384,0.03550399939219157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,16384,0.030269867181777953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,12288,0.03429333368937175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,12288,0.03067520062128703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,12288,0.02395413319269816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,10240,0.02934933304786682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,8192,0.024292266368865965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,10240,0.02805759906768799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,8192,0.025146667162577314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,8192,0.01760106682777405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,7168,0.02190399964650472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,256,0.0048885335524876915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,7168,0.023886932929356893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,6144,0.019257599115371705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,7168,0.016169599692026772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,6144,0.01474666694800059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,5120,0.0170741339524587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,6144,0.02267306645711263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,5120,0.01304533382256826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,5120,0.021434666713078816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,4096,0.013339733084042868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,3584,0.01213759978612264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,4096,0.02001813252766927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,32,0.0027456000447273255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,3584,0.01104213297367096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,3584,0.019429334004720054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,3072,0.00963200032711029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,3072,0.01895786722501119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,2560,0.009594666957855224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,2560,0.018221867084503175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,2560,0.009166933099428813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,2048,0.008353066444396973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,2048,0.008193066716194153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,2048,0.015811199943224587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,1536,0.007202133536338806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,1536,0.015589333573977151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,1536,0.007034666836261749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,1024,0.004558933277924856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,1024,0.005734399954477946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,1024,0.015401599804560342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,768,0.003941333293914795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,51200,0.08080960114796956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,2048,0.008113066852092742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,768,0.01511253317197164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,512,0.003509333233038584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,512,0.00528106689453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,256,0.003155199935038885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,256,0.004997333387533823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,256,0.014562132954597472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,128,0.002926933268706004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,128,0.004693333307902018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,128,0.014532267053922018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,64,0.0027295999228954316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,32,0.002784000088771184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,64,0.014626133441925048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,32,0.014506666858990987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,65536,0.14240105946858722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,65536,0.08562560081481933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,4096,0.011603200435638427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,3072,0.010872532924016316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,51200,0.11255146662394207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,51200,0.07034239768981934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,65536,0.09416746298472087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,16384,0.03918186823527019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,16384,0.0327839990456899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,12288,0.0308896005153656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,16384,0.027729066212972005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,51200,0.07532266775767008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,10240,0.026200532913208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,12288,0.02221226692199707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,12288,0.028628265857696532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,10240,0.02601813276608785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,8192,0.021658666928609214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,10240,0.01983786622683207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,8192,0.01665066679318746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,7168,0.019640533129374187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,6144,0.017313067118326822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,7168,0.015326933066050211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,7168,0.02334933280944824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,5120,0.015381333231925965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,6144,0.013910399874051413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,6144,0.022168533007303873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,5120,0.0125408003727595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,4096,0.013658666610717773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,5120,0.020666666825612388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,3584,0.012679466605186462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,4096,0.010985599954922994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,768,0.005343999962011973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,3584,0.010598400235176086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,3072,0.01123199959595998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,3584,0.01914986570676168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,512,0.014894933501879374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,2560,0.00902826686700185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,3072,0.009308800101280212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,3072,0.018598399559656777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,2560,0.01773653427759806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,2560,0.008689066767692566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,2048,0.0077567999561627705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,2048,0.01572266618410746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,1536,0.006788266698519389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,1536,0.015131733814875283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,1536,0.0065642664829889934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,1024,0.005975466469923655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,1024,0.015144532918930054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,768,0.003945599993069967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,768,0.005491200089454651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,512,0.0035018667578697203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,512,0.014800000190734863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,10240,0.021112533410390218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,512,0.005151999990145365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,256,0.01448853313922882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,256,0.0049002667268117275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,128,0.002935466667016347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,128,0.014381866653760275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,128,0.004675200084845225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,8192,0.024753065903981526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,64,0.0027146667242050173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,64,0.014679466684659323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,32,0.002738133321205775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,32,0.01458133359750112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,4096,0.019694934288660683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,65536,0.11857386430104573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,65536,0.07379199663798014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,2048,0.007551999886830647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,1024,0.0043722664316495265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,51200,0.09422079722086588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,65536,0.09343893527984619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,51200,0.05957440137863159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,16384,0.03322666684786479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,51200,0.07388052940368653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,16384,0.027395200729370118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,16384,0.029824000597000123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,768,0.014906666676203408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,12288,0.0260863999525706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,10240,0.022381865978240968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,12288,0.021996800104777017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,10240,0.024406399329503378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,8192,0.0189301331837972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,10240,0.017705599466959633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,8192,0.01637333333492279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,8192,0.02262293299039205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,7168,0.017283199230829875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,6144,0.015219199657440185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,7168,0.014235732952753702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,7168,0.021744000911712646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,256,0.0031680000325044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,6144,0.013658666610717773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,5120,0.013592533270517983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,4096,0.011756799618403117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,5120,0.012357333302497863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,5120,0.020023467143376668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,3584,0.010889599720637005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,4096,0.010805333654085796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,4096,0.018824533621470133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,3072,0.010084266463915508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,3584,0.00995733340581258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,3584,0.01853333314259847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,3072,0.009317333499590557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,2560,0.00827519992987315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,3072,0.0175872008005778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,2048,0.007145600020885467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,2560,0.008261333405971526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,2560,0.01628159979979197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,2048,0.01601599951585134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,1536,0.0064416001240412395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,1536,0.015657599767049155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,1024,0.00584853341182073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,1024,0.015067733327547708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,768,0.0038880000511805216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,768,0.014924800395965577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,768,0.005618133147557576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,512,0.003421866645415624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,512,0.014938666423161825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,512,0.005063466727733612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,256,0.0030965333183606463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,256,0.014551466703414917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,128,0.002810666710138321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,128,0.01437440017859141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,128,0.0046741331617037455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,64,0.0026581334571043652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,64,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,32,0.002679466704527537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,32,0.014459733168284097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,12288,0.025803732872009277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,65536,0.09601173400878907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,65536,0.060456534226735435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,51200,0.07437866528828939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,65536,0.08347520033518473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,51200,0.05072426795959473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,51200,0.06607253154118856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,16384,0.026898133754730224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,2048,0.007225599884986877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,1536,0.005729066828886667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,16384,0.026548266410827637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,1024,0.00420053352912267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,12288,0.02375146746635437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,10240,0.018685867389043175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,10240,0.017874133586883546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,8192,0.015675733486811318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,8192,0.021285333236058555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,8192,0.015359999736150107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,7168,0.014327466487884521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,7168,0.02058346668879191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,7168,0.014215466380119324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,6144,0.012865066528320312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,6144,0.019789866606394448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,6144,0.012946133812268576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,5120,0.011754666765530903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,5120,0.011725866794586181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,5120,0.019394133488337198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,4096,0.010244266192118327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,4096,0.0184170663356781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,4096,0.010214400291442872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,3584,0.00955839951833089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,3584,0.017668267091115318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,3584,0.009379200140635173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,3072,0.008889599641164144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,6144,0.02086720069249471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,3072,0.01665386656920115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,3072,0.008686932921409606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,2560,0.007547733187675476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,2560,0.007910400132338206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,2560,0.01598186691602071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,2048,0.006681600213050842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,16384,0.0254037340482076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,2048,0.007009066641330719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,12288,0.021576533714930214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,1536,0.004905599852403005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,12288,0.019803732633590698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,1024,0.00405973345041275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,1536,0.015539200107256571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,1024,0.0055285334587097164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,1024,0.0151146670182546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,768,0.005313066641489664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,768,0.01498133341471354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,512,0.0033919999996821085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,512,0.004950400193532308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,256,0.002980266759792964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,256,0.014524799585342408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,256,0.004760533571243286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,128,0.0027434666951497394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,128,0.014387200276056925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,128,0.004514133433500925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,64,0.0026506667335828146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,64,0.014365866780281067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,32,0.002677333354949951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,32,0.014367999633153281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,256,0.004851200183232625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,65536,0.07361599604288736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,65536,0.04994560082753499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,65536,0.08345279693603516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,51200,0.05860373179117838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,51200,0.04281066656112671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,2048,0.015660799543062844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,16384,0.02162453333536784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,16384,0.0247488001982371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,16384,0.023975465695063272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,768,0.003634133438269297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,12288,0.017499732971191406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,12288,0.02168533404668172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,10240,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,12288,0.01967680056889852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,10240,0.02087786595026652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,10240,0.017436800400416057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,512,0.014722133676211039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,8192,0.014958932995796204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,7168,0.0121888001759847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,8192,0.01971413294474284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,6144,0.010924800237019857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,7168,0.019326933224995933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,6144,0.012516267100969949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,6144,0.01889066696166992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,5120,0.00997866690158844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,5120,0.018773333231608073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,5120,0.011303466558456422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,4096,0.008873599767684936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,4096,0.00956160028775533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,4096,0.017107200622558594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,3584,0.00853546659151713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,3584,0.016691199938456216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,3584,0.009010133147239686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,3072,0.007977599898974102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,3072,0.016295466820398966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,3072,0.007971199850241344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,2560,0.007241599758466085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,2560,0.016081066926320393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,2560,0.007597866654396057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,2048,0.0057888001203536986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,2048,0.015824000040690102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,1536,0.004706133405367533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,1536,0.015396266182263692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,1536,0.006445866823196411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,1024,0.004119466741879781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,1024,0.015103999773661295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,1024,0.005654400090376536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,10240,0.02262826760609945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,768,0.0036906667053699495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,768,0.005339733262856802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,768,0.015064533551534018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,512,0.0034282666941483817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,8192,0.01309333344300588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,512,0.014830933014551798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,512,0.005020800232887268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,256,0.004829866687456766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,256,0.014615466197331747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,128,0.0028042666614055633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,1536,0.006385066608587901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,64,0.002700799951950709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,128,0.014514133334159851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,64,0.014490666985511779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,32,0.0027114666998386385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,32,0.01451520025730133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,65536,0.049243732293446855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,51200,0.03960959911346436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,65536,0.08338560263315836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,51200,0.03349866469701131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,51200,0.06612693468729655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,16384,0.01584106683731079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,16384,0.021222400665283202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,2048,0.006758399804433187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,16384,0.024485333760579427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,12288,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,12288,0.019619200627009073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,12288,0.019641600052515664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,10240,0.011849600076675414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,10240,0.017357865969340004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,10240,0.01902186671892802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,8192,0.010291199882825215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,8192,0.017802667617797852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,8192,0.014801067113876343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,7168,0.009733333190282186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,7168,0.017748266458511353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,7168,0.013185066978136697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,6144,0.008785067001978557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,6144,0.017359999815622966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,5120,0.008317866424719492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,256,0.0029909332593282064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,5120,0.01809599995613098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,51200,0.06638186772664388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,5120,0.010279466708501179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,4096,0.009194667140642803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,4096,0.01738026738166809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,128,0.00470719983180364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,3584,0.006806399921576183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,3584,0.00885653297106425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,3072,0.00631466656923294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,3584,0.016951467593510947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,2560,0.006117333471775055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,65536,0.03863146702448527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,3072,0.007863466441631318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,3072,0.0166293332974116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,2048,0.005367466807365417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,2560,0.007530666887760162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,2560,0.016201600432395935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,2048,0.01590933303038279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,2048,0.006715733309586842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,1536,0.004682666560014089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,1024,0.003947733342647553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,1536,0.00631039987007777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,1536,0.015660799543062844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,1024,0.015442132949829102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,1024,0.005579733351866404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,512,0.0033813332517941795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,768,0.015103999773661295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,512,0.005009066561857859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,512,0.014729600151379904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,256,0.0030069333811601003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,256,0.014487466216087342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,256,0.004823466638724009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,128,0.00276053324341774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,128,0.014765866597493491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,128,0.00468800018231074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,64,0.0030154667794704436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,32,0.003044266750415166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,64,0.014619732896486918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,65536,0.03833920160929362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,65536,0.033225599924723306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,65536,0.08248000144958496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,51200,0.031310933828353885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,51200,0.029312000672022505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,4096,0.007229866584142049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,51200,0.06538133223851522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,16384,0.013193600376447043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,16384,0.024461867411931355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,12288,0.011222400267918905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,12288,0.018428800503412883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,12288,0.019615999857584634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,10240,0.010188800096511842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,7168,0.013733333349227906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,768,0.003734400123357773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,10240,0.01810773412386576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,10240,0.01699626644452413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,8192,0.009171199798583985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,768,0.00527999997138977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,7168,0.008478933572769165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,8192,0.014286933342615762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,8192,0.017438934246699015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,6144,0.007774933179219563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,6144,0.01728960076967875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,6144,0.011570133765538533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,5120,0.007111466427644093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,5120,0.01781439979871114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,4096,0.006026666859785716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,4096,0.017054933309555053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,4096,0.009248000383377076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,32,0.01474346617857615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,3584,0.006442666550477346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,3584,0.008881066242853801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,3584,0.016909867525100708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,3072,0.005958400170008342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,3072,0.00795306662718455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,2560,0.006077866752942403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,2560,0.01606933375199636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,2048,0.005376000205675761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,2560,0.007660800218582153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,2048,0.015797332922617594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,2048,0.006727466483910878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,1536,0.0047189335028330484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,6144,0.011832533280054729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,16384,0.01986879905064901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,1536,0.015550933281580605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,1024,0.004011733333269755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,1024,0.005576533575852713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,1024,0.015197867155075073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,768,0.003583999971548716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,768,0.01530239979426066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,768,0.00525546669960022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,512,0.003402666747570038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,512,0.004948266843954722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,512,0.015038933356602988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,256,0.0032511999209721885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,256,0.01470080018043518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,256,0.004795733094215393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,7168,0.017864533265431724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,128,0.002883200099070867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,64,0.0027413333455721537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,128,0.014495999614397685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,32,0.002844800055027008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,5120,0.010435199737548828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,64,0.01453013320763906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,32,0.014470400412877402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,65536,0.02712106704711914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,65536,0.0271616001923879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,51200,0.022418133417765298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,51200,0.024462932348251344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,65536,0.08299199740091959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,16384,0.01039573351542155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,51200,0.06575680176417033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,16384,0.018438400824864705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,12288,0.00897813340028127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,3072,0.01667413314183553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,12288,0.01695573329925537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,12288,0.01921280026435852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,10240,0.016547200083732606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,10240,0.017513600985209148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,8192,0.007786666850248973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,8192,0.01725226640701294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,8192,0.01406826674938202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,7168,0.017528533935546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,7168,0.012836266557375589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,6144,0.006065066655476888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,6144,0.017256534099578856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,6144,0.011516799529393513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,5120,0.006401066482067108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,5120,0.017854932943979898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,5120,0.010460799932479859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,4096,0.00582826683918635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,7168,0.01297706663608551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,4096,0.00918506681919098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,4096,0.017214934031168617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,128,0.004679466784000397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,3584,0.006281599899133046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,3584,0.008782933155695598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,3072,0.005950933198134104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,3584,0.0167797327041626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,2560,0.006144000093142191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,3072,0.007895466685295106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,3072,0.016771199305852254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,2048,0.0054613331953684485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,2560,0.016184533635775246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,2048,0.016035200158754984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,2048,0.006712533533573151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,1536,0.0047872001926104225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,1536,0.015597866972287497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,1536,0.006246399879455566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,1024,0.004112000018358231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,1024,0.005555200080076853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,10240,0.00869546631971995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,1024,0.015438933173815408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,768,0.003786666691303253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,512,0.0034933333595593774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,768,0.005362133185068766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,768,0.015093333522478738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,7168,0.006671999891599019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,512,0.004902400076389313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,256,0.0031210665901501974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,512,0.015041066209475198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,256,0.0047989333669344585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,128,0.0028319999575614927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,256,0.014612266421318054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,64,0.002739199995994568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,32,0.0027669332921504974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,128,0.0045855998992919925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,128,0.01460693379243215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,64,0.014689067006111145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,51200,0.014205867052078247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,65536,0.02229866584142049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,51200,0.0207370658715566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,51200,0.065611732006073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,16384,0.007582933207352956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,1536,0.006236800054709116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,16384,0.01731946667035421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,12288,0.00631466656923294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,12288,0.017179733514785765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,10240,0.00628053347269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,2560,0.007624533275763195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,10240,0.017463467518488564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,10240,0.0164490669965744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,8192,0.005965866645177205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,8192,0.017002665996551515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,8192,0.014055466651916504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,7168,0.006000000238418579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,7168,0.01747200091679891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,16384,0.024013866980870567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,6144,0.005769599974155426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,65536,0.016990933815638223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,6144,0.01695466637611389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,65536,0.08257386684417725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,5120,0.006089599927266439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,5120,0.010262399911880493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,5120,0.017829332749048868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,4096,0.005714133381843567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,4096,0.017205333709716795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,4096,0.009037866195042928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,12288,0.018773333231608073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,3584,0.006248533229033152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,3584,0.016832000017166136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,3072,0.005919999877611796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,3584,0.00863039990266164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,3072,0.01646080017089844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,2560,0.0060586666067441305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,3072,0.007921066880226136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,2048,0.005432533224423727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,2560,0.007514666517575581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,2560,0.01627946694691976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,1536,0.004612266520659129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,2048,0.0065994665026664736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,2048,0.015803733468055726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,1024,0.00405973345041275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,1536,0.01560640037059784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,1024,0.005478399991989136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,1024,0.015235199530919393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,768,0.003671466559171677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,768,0.014877866705258688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,512,0.0033791999022165934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,512,0.014666666587193808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,512,0.0048981333772341405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,256,0.0029557332396507262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,256,0.014525866508483887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,256,0.004603733122348785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,128,0.0028031999866167706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,32,0.014497066537539164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,128,0.0144896000623703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,128,0.00450133333603541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,6144,0.011544533570607503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,32,0.0027114666998386385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,64,0.014428800344467163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,16384,0.023804799715677897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,65536,0.01141973336537679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,32,0.014382933576901754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,51200,0.009854933619499207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,65536,0.01875306765238444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,65536,0.08200960159301758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,16384,0.005902933577696482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,51200,0.06504640181859335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,16384,0.023592533667882283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,12288,0.005840000013510386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,16384,0.01725546717643738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,10240,0.006000000238418579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,12288,0.016938666502634682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,10240,0.016357333461443583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,10240,0.017197867234547935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,8192,0.005876266459623972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,1536,0.006230400005976359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,8192,0.016785067319869996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,8192,0.01390720009803772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,7168,0.005789866546789805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,7168,0.012654933333396911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,6144,0.00565119981765747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,7168,0.01722453236579895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,5120,0.006021333237489065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,6144,0.01698026657104492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,768,0.005227733155091604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,5120,0.010238933563232421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,4096,0.005520000060399374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,5120,0.017391999562581383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,3584,0.006097066899140676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,4096,0.00906773308912913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,4096,0.016777600844701132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,3072,0.005686399837334951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,3584,0.016495999693870545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,3072,0.016150400042533875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,2560,0.0057536001006762184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,2560,0.01585706671079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,2560,0.007490133245786031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,2048,0.005116799970467886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,2048,0.015603199601173401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,7168,0.012851199507713318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,1536,0.004468266665935516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,1536,0.015615999698638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,1536,0.006198399762312571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,1024,0.0038218667109807336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,51200,0.017767467101415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,1024,0.015185067057609558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,1024,0.005524266759554545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,768,0.003634133438269297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,12288,0.018755199511845906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,768,0.014994133512179056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,512,0.0033098667860031127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,512,0.014814933141072592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,512,0.004875733455022176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,256,0.0030389333764712016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,256,0.01446399986743927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,6144,0.01144426663716634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,256,0.004724266628424326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,128,0.014380799730618796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,128,0.004569600025812784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,64,0.0026176000634829206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,3072,0.007880533238252004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,64,0.01448319951693217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,32,0.002681600054105123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,32,0.014475733041763306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,65536,0.017832533518473307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,51200,0.007612800101439159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,51200,0.017729065815607705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,64,0.002666666607062022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,16384,0.005670399963855743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,2048,0.006658133367697399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,12288,0.005662933488686879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,16384,0.01696959932645162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,10240,0.005842133363087972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,12288,0.017130666971206666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,8192,0.005786666770776113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,10240,0.017146666844685875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,7168,0.0056533331672350565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,8192,0.01681706706682841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,6144,0.005605333546797434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,7168,0.017257599035898845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,6144,0.016919465859731038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,4096,0.00552106648683548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,5120,0.017197867234547935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,3584,0.005983999868233999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,4096,0.01672533353169759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,768,0.00514026681582133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,3584,0.01629866659641266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,2560,0.00561599979797999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,3072,0.01602666676044464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,2048,0.004982399940490723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,2560,0.015612799922625223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,1536,0.004411733150482178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,1536,0.015130666891733804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,128,0.0028384000062942503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,1024,0.0038463999827702843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,768,0.0034805332620938623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,1024,0.015077333648999533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,512,0.0032106667757034303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,768,0.014541866381963095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,256,0.003014400104681651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,512,0.014669866363207499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,128,0.002825599908828735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,256,0.014475733041763306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,64,0.0026591998835404714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,65536,0.008825600147247314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,128,0.014469333489735923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,32,0.0026367999613285064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,64,0.014377599954605103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,65536,0.007793066898981731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,32,0.014387200276056925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,51200,0.007667199770609538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,65536,0.018156800667444864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,16384,0.00572266678015391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,51200,0.01856000026067098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,3584,0.008687999844551087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,12288,0.005780266722043356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,16384,0.017063466707865398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,12288,0.01689599951108297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,8192,0.0057322666049003605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,10240,0.017090133825937905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,7168,0.005614933371543884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,8192,0.016898133357365928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,6144,0.005514666438102722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,7168,0.017128533124923705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,5120,0.005897599955399831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,6144,0.017040000359217326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,5120,0.0059456000725428265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,5120,0.017153066396713258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,4096,0.005432533224423727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,3584,0.005990399916966756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,4096,0.01687893271446228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,3072,0.005648000041643778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,3584,0.016357333461443583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,2560,0.005588266750176748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,3072,0.01618026693662008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,2048,0.005017599960168203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,1536,0.00447573314110438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,2048,0.015763200322786965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,3072,0.005644799768924713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,1024,0.0038346665600935614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,1536,0.015280000368754067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,768,0.0034645333886146545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,512,0.00323840007185936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,1024,0.015161599715550741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,768,0.014784000317255654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,256,0.0028437333802382152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,128,0.002703999976317088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,512,0.014653866489728292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,256,0.014375467101732889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,64,0.0025631998976071674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,128,0.014339199662208557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,2048,0.015365333358446757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,32,0.0025813333690166474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,32,0.0143477330605189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,64,0.014366933703422546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,16384,0.0005077333189547062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,12288,0.0005215999980767567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,16384,0.0019658666104078295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,10240,0.0005013333323101203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,10240,0.005771733323733012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,12288,0.001889066646496455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,10240,0.0019551999866962433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,7168,0.0004767999984323978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,51200,0.0005472000067432721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,8192,0.0018986667195955913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,51200,0.0019519999623298645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,6144,0.0004650666688879331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,2560,0.01567466656366984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,7168,0.001911466692884763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,5120,0.00047146665553251903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,3584,0.00048106666654348376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,5120,0.0019178666174411775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,3072,0.0004896000027656555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,2560,0.0004917333212991555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,6144,0.0019178666174411775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,3584,0.0019829332828521728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,3072,0.0019402666638294857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,4096,0.0005215999980767567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,2048,0.000549333356320858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,1536,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,2048,0.001911466692884763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,2560,0.00191893329222997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,1024,0.0005226666728655497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,1536,0.001956266661485036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,768,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,1024,0.001972266659140587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,768,0.0019466667125622433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,256,0.0004959999894102415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,128,0.0004992000137766202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,512,0.001960533360640208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,256,0.0019829332828521728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,128,0.001932799940307935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,32,0.0005258666972319285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,8192,0.00048106666654348376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,65536,0.0004970666641990344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,65536,4096,0.0018709332992633183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,51200,0.0005013333323101203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,65536,0.0020106667031844455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,512,0.0005215999980767567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,12288,0.0005215999980767567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,65536,64,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,16384,0.0004639999940991402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,16384,0.0019082666685183841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,8192,0.0005152000114321709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,7168,0.0005621333296100298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,10240,0.0005013333323101203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,10240,0.001975466683506966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,8192,0.0019359999646743138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,51200,0.001889066646496455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,6144,0.000508799993743499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,7168,0.0018879999717076618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,5120,0.0005098666685322921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,4096,0.00048533333465456964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,3072,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,5120,0.0019029332945744199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,3584,0.0004927999960879485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,4096,0.0018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,3072,0.0018986667195955913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,2560,0.0004981333389878273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,3584,0.0018922666708628337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,2560,0.0018826667219400406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,1536,0.000513066661854585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,1024,0.0004896000027656555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,2048,0.001886933296918869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,768,0.000513066661854585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,1536,0.0019296000401178997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,1024,0.0019594666858514147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,768,0.0019253333409627277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,256,0.0005472000067432721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,512,0.0019626667102177935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,128,0.00047999999175469085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,256,0.0019690666347742082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,64,0.0005002666575213273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,128,0.0019829332828521728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,32,0.00048746665318806964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,65536,0.000514133336643378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,51200,0.00048746665318806964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,65536,0.001970133309563001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,16384,0.00048533333465456964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,12288,0.000513066661854585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,51200,0.001985066632429759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,12288,0.0019424000134070714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,10240,0.0004767999984323978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,12288,0.0019648000597953796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,8192,0.0018901333212852478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,8192,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,10240,0.0018933333456516265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,7168,0.00048640000944336255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,7168,0.0018816000471512477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,6144,0.001915733392039935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,51200,6144,0.0019797333826621372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,2048,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,5120,0.0019402666638294857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,4096,0.0005119999870657921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,3584,0.0005119999870657921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,4096,0.0019861333072185515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,3072,0.0005098666685322921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,3584,0.0019317333896954852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,2560,0.000513066661854585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,3072,0.002001066630085309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,2048,0.0004949333456655343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,2560,0.0019744000087181727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,2048,0.0019551999866962433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,1024,0.0004927999960879485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,1536,0.0019125333676735562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,1024,0.0019338666150967281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,768,0.0019274666905403138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,512,0.0005034666818877061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,512,0.001983999957640966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,256,0.0004927999960879485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,256,0.001995733380317688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,128,0.0004896000027656555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,128,0.0019359999646743138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,64,0.000516266655176878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,32,0.00048640000944336255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,16384,16384,0.0019456000377734502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,65536,0.00048533333465456964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,65536,0.0018911999960740406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,6144,0.000549333356320858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,51200,0.0005077333189547062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,5120,0.00048106666654348376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,16384,0.0006720000257094701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,12288,0.0007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,16384,0.0019082666685183841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,51200,0.0018997333943843841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,10240,0.000508799993743499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,12288,0.0018976000448067982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,51200,512,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,8192,0.00047040001178781193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,7168,0.00046719998742143315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,8192,0.001876266673207283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,6144,0.0005002666575213273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,1536,0.0005749333028992017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,10240,0.0019082666685183841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,6144,0.001971199984351794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,4096,0.0004970666641990344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,7168,0.0019541333119074506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,5120,0.0020245333512624105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,3584,0.0004981333389878273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,3072,0.0004949333456655343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,3584,0.001961600035429001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,16384,768,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,2560,0.00048319998507698376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,3072,0.0019573333362738294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,2048,0.000508799993743499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,1536,0.00047573332364360496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,2048,0.0018922666708628337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,1536,0.0019253333409627277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,768,0.0005184000047544639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,1024,0.0019178666174411775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,512,0.0004896000027656555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,768,0.0019317333896954852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,256,0.0004917333212991555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,512,0.0018805333723624547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,128,0.0005173333299656709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,256,0.0019765333582957584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,64,0.0005119999870657921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,128,0.001960533360640208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,32,0.0004906666775544484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,51200,0.0004981333389878273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,65536,0.0005194666795432567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,65536,0.001956266661485036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,16384,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,12288,0.00047360000511010485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,16384,0.001915733392039935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,10240,0.000506666675209999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,12288,0.0018879999717076618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,8192,0.0004767999984323978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,10240,0.0018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,7168,0.0004959999894102415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,8192,0.0018826667219400406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,6144,0.00048746665318806964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,7168,0.0019626667102177935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,5120,0.0005077333189547062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,6144,0.0019050666441520054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,4096,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,5120,0.0019744000087181727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,3584,0.0004949333456655343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,4096,0.0019445333629846573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,3072,0.0005098666685322921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,3584,0.0019456000377734502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,5120,0.0004959999894102415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,2560,0.0004981333389878273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,3072,0.001886933296918869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,2048,0.00048106666654348376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,2560,0.001932799940307935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,1536,0.000510933343321085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,2048,0.001960533360640208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,2560,0.0019498666127522786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,1024,0.00047360000511010485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,1536,0.0019434666881958645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,12288,1024,0.0004522666645546754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,768,0.00048533333465456964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,1024,0.0018816000471512477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,512,0.0004917333212991555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,768,0.0019573333362738294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,256,0.0004917333212991555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,128,0.0004821333413322766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,256,0.0018911999960740406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,64,0.0005205333232879639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,128,0.001889066646496455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,10240,32,0.0004885333279768627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,65536,0.00047146665553251903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,51200,0.0004949333456655343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,65536,0.0018922666708628337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,16384,0.0004650666688879331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,12288,0.00048106666654348376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,16384,0.001884799947341283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,51200,0.001865600049495697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,10240,0.0004896000027656555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,12288,0.0018954666952292125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,8192,0.0005077333189547062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,51200,0.0019434666881958645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,7168,0.0004746666798988978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,8192,0.0018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,10240,0.0019029332945744199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,6144,0.00047893334800998375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,5120,0.00047999999175469085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,7168,0.0020106667031844455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,5120,0.0020703999946514767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,4096,0.00047786667322119073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,3584,0.0005119999870657921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,3584,0.0019456000377734502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,3072,0.0005343999713659286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,2560,0.0004885333279768627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,3072,0.002019199977318446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,2048,0.0005184000047544639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,2560,0.0019456000377734502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,1536,0.0004959999894102415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,2048,0.001934933289885521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,1024,0.0005173333299656709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,1536,0.0019519999623298645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,768,0.00048640000944336255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,1024,0.001985066632429759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,512,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,12288,4096,0.00198186660806338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,512,0.001961600035429001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,256,0.0005429333075881004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,128,0.0005397333453098933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,256,0.0019413333386182785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,64,0.0005322666838765144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,128,0.0018826667219400406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,8192,32,0.0005279999847213428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,51200,0.0005365333209435146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,65536,0.00048106666654348376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,16384,0.0005408000200986862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,65536,0.0018997333943843841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,12288,0.0005152000114321709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,51200,0.0019509332875410716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,16384,0.0018858666221300761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,10240,0.0005077333189547062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,12288,0.0019231999913851422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,10240,0.0019381333142518998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,8192,0.001972266659140587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,7168,0.0005013333323101203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,7168,0.0019690666347742082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,6144,0.0019498666127522786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,6144,0.0004970666641990344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,5120,0.0004927999960879485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,6144,0.0018986667195955913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,4096,0.0019594666858514147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,4096,0.0004927999960879485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,5120,0.0018879999717076618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,3584,0.000516266655176878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,4096,0.0019498666127522786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,3072,0.0004992000137766202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,3584,0.0019658666104078295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,2560,0.0004970666641990344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,3072,0.0019359999646743138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,2048,0.0004970666641990344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,2560,0.001878400022784869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,1536,0.000514133336643378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,1536,0.0018805333723624547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,1024,0.0005013333323101203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,768,0.000508799993743499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,8192,768,0.0020224000016848247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,768,0.0018944000204404194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,512,0.0005333333586653073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,256,0.0004896000027656555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,512,0.001985066632429759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,128,0.00048640000944336255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,256,0.001934933289885521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,64,0.0004949333456655343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,128,0.0019519999623298645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,10240,512,0.0018837332725524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,32,0.0004906666775544484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,65536,0.000516266655176878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,51200,0.0006218666831652324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,65536,0.0019744000087181727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,16384,0.0004927999960879485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,51200,0.0019744000087181727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,12288,0.000508799993743499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,12288,0.001884799947341283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,10240,0.0005013333323101203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,8192,0.00048746665318806964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,10240,0.0019573333362738294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,7168,0.0005119999870657921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,8192,0.0019050666441520054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,7168,8192,0.0004970666641990344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,6144,0.000516266655176878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,7168,0.001963733385006587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,5120,0.0004885333279768627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,6144,0.0018933333456516265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,4096,0.00048640000944336255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,5120,0.0019061333189407986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,3584,0.0005365333209435146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,4096,0.0018976000448067982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,3072,0.00048746665318806964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,2560,0.0005056000004212062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,3072,0.0019487999379634857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,2048,0.0004906666775544484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,2560,0.0019999999552965165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,1536,0.0005045333256324132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,2048,0.0019690666347742082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,1024,0.000549333356320858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,1536,0.001937066639463107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,768,0.0005215999980767567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,1024,0.0019391999890406926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,512,0.0004767999984323978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,768,0.002005333329240481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,256,0.0004896000027656555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,256,0.001874133323629697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,128,0.0004842666598657767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,2048,0.0019221333165963489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,128,0.0018965333700180053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,64,0.00047999999175469085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,7168,1024,0.001967999959985415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,6144,32,0.0008021333565314612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,65536,0.0004896000027656555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,51200,0.0004746666798988978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,65536,0.0019765333582957584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,16384,0.00048746665318806964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,51200,0.0019231999913851422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,16384,0.0019231999913851422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,10240,0.000506666675209999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,12288,0.001926400015751521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,8192,0.0004767999984323978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,10240,0.0019797333826621372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,7168,0.0004917333212991555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,8192,0.001966933285196622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,6144,0.0004970666641990344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,7168,0.002038399999340375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,5120,0.0004949333456655343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,6144,0.00191040001809597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,16384,0.0019381333142518998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,4096,0.00048746665318806964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,5120,0.0018976000448067982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,3584,0.0004842666598657767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,4096,0.0019061333189407986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,3072,0.0004992000137766202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,3584,0.001934933289885521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,2560,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,3072,0.0019167999426523846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,2048,0.0004746666798988978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,2560,0.001971199984351794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,1536,0.0004959999894102415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,2048,0.0019231999913851422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,1024,0.0005098666685322921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,1536,0.0019658666104078295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,768,0.000506666675209999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,1024,0.0018794666975736618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,512,0.00047999999175469085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,768,0.0019914666811625165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,256,0.0004981333389878273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,512,0.0018944000204404194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,128,0.000514133336643378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,256,0.0019733333339293797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,5120,128,0.0019445333629846573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,64,0.00047786667322119073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,32,0.0004949333456655343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,65536,0.00047360000511010485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,51200,0.0005205333232879639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,65536,0.0018837332725524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,16384,0.0004885333279768627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,12288,0.0004917333212991555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,16384,0.001884799947341283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,51200,0.0019391999890406926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,10240,0.00047573332364360496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,12288,0.0020234666764736174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,8192,0.0004981333389878273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,10240,0.0018698666244745255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,7168,0.0005173333299656709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,8192,0.001889066646496455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,6144,0.0004885333279768627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,7168,0.0019893333315849303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,5120,0.0004949333456655343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,6144,0.001971199984351794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,4096,0.000506666675209999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,5120,0.0019413333386182785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,4096,0.0019093333433071774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,3584,0.0005205333232879639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,3072,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,3584,0.0019039999693632125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,5120,12288,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,2560,0.00048533333465456964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,3072,0.0019776000330845515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,2048,0.0005301333342989285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,2560,0.0019391999890406926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,2048,0.0018858666221300761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,1024,0.0004906666775544484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,1536,0.0019456000377734502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,768,0.0004992000137766202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,1024,0.0018698666244745255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,512,0.0005237333476543427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,768,0.0019594666858514147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,256,0.0005365333209435146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,512,0.0019125333676735562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,128,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,256,0.0019061333189407986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,64,0.0004906666775544484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,4096,128,0.0019744000087181727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,32,0.0005333333586653073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,65536,0.0004842666598657767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,51200,0.00048106666654348376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,65536,0.0019253333409627277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,16384,0.0004917333212991555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,12288,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,16384,0.001983999957640966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,10240,0.0005482666815320651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,51200,0.001963733385006587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,12288,0.001970133309563001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,8192,0.00046080000077684715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,10240,0.0019829332828521728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,8192,0.0019210666418075562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,6144,0.0005450666571656863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,5120,0.0005002666575213273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,6144,0.0019082666685183841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,4096,0.0004767999984323978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,5120,0.001876266673207283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,3584,0.0005258666972319285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,3584,0.0018911999960740406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,512,0.0019648000597953796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,3072,0.00048746665318806964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,3072,0.0018911999960740406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,2560,0.0005333333586653073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,2048,0.0004896000027656555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,2560,0.0019487999379634857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,1536,0.0004906666775544484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,1536,0.0019786667078733446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,1024,0.0004992000137766202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,1024,0.0019573333362738294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,768,0.0005013333323101203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,768,0.0019551999866962433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,512,0.0009162666896979013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,4096,1536,0.0005002666575213273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,512,0.0018858666221300761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,256,0.0005290666595101357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,128,0.0005034666818877061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,256,0.0019658666104078295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,128,0.0019424000134070714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,64,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,32,0.00048640000944336255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,65536,0.00048746665318806964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3584,7168,0.0004992000137766202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,6144,3584,0.0018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,51200,0.0004917333212991555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,16384,0.000513066661854585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,51200,0.0019648000597953796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,4096,0.0019050666441520054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,16384,0.00186666672428449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,12288,0.0004959999894102415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,10240,0.0005056000004212062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,12288,0.001926400015751521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,8192,0.0004927999960879485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,10240,0.0019541333119074506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,7168,0.00048533333465456964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,8192,0.0019434666881958645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,6144,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,7168,0.0018976000448067982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,5120,0.000508799993743499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,6144,0.0019509332875410716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,4096,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,5120,0.0019221333165963489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,2048,0.0019285333653291064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,3584,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,3072,0.0005056000004212062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,3584,0.002001066630085309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,2560,0.000513066661854585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,3072,0.0018816000471512477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,2048,0.0007818666597207387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,2560,0.0018698666244745255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,1536,0.0005653333539764086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,2048,0.0019136000424623489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,1024,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,1536,0.002004266654451688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,768,0.0005002666575213273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,1024,0.001884799947341283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,512,0.0005279999847213428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,768,0.001961600035429001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,256,0.0005194666795432567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,512,0.001924266666173935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,128,0.0005152000114321709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,256,0.001963733385006587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,64,0.0005397333453098933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,3072,32,0.00048533333465456964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,65536,0.0005205333232879639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,51200,0.0005056000004212062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,65536,0.0019733333339293797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,16384,0.00047146665553251903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,51200,0.0018837332725524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,16384,0.00191893329222997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,10240,0.00047146665553251903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,12288,0.001886933296918869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,8192,0.0005002666575213273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,10240,0.0018965333700180053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,8192,0.0019765333582957584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,7168,0.00048533333465456964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,6144,0.0004821333413322766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,7168,0.0019626667102177935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,65536,0.0018922666708628337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,6144,0.0019765333582957584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3584,7168,0.002014933278163274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,5120,0.0005013333323101203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,5120,0.0019413333386182785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,3584,0.0004842666598657767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,4096,0.00186666672428449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,3072,0.0004906666775544484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,3584,0.0018634666999181111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,2560,0.00047893334800998375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,2560,0.0018794666975736618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,2048,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,2048,0.0019466667125622433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,1536,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,4096,0.0019381333142518998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,1536,0.0019317333896954852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,1024,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,768,0.000514133336643378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,1024,0.0018773333479960759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,768,0.0018879999717076618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,512,0.001956266661485036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,256,0.000514133336643378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,256,0.0019648000597953796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,128,0.0005205333232879639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,3072,128,0.0019530666371186574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,64,0.0005098666685322921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,128,0.0019498666127522786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,32,0.0005173333299656709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,65536,0.0004992000137766202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,51200,0.0005034666818877061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,65536,0.0019626667102177935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,12288,0.000516266655176878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,16384,0.00047999999175469085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,51200,0.001889066646496455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,12288,0.0005184000047544639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,16384,0.0018986667195955913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,10240,0.0005205333232879639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,8192,0.0005386666705211004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,10240,0.0018976000448067982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,8192,0.0019296000401178997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,7168,0.0005386666705211004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,6144,0.0004885333279768627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,7168,0.002128000060717265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,4096,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,5120,0.000514133336643378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,4096,0.000514133336643378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,5120,0.002006400004029274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,3584,0.000508799993743499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2560,3072,0.0019231999913851422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,3072,0.0004896000027656555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,3584,0.001960533360640208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,2560,0.0005013333323101203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,3072,0.001970133309563001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,2048,0.0005045333256324132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,2048,0.001886933296918869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,1536,0.0004949333456655343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,1024,0.00047999999175469085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,1024,0.0019317333896954852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2560,512,0.0004970666641990344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,768,0.0005013333323101203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,512,0.000506666675209999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,768,0.0019744000087181727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,512,0.001971199984351794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,256,0.0005034666818877061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,128,0.0005215999980767567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,128,0.0018944000204404194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,64,0.0005045333256324132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,2048,32,0.0005098666685322921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,65536,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,65536,0.0018986667195955913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,12288,0.0018719999740521114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,51200,0.000510933343321085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,51200,0.0018805333723624547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,16384,0.0004992000137766202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,16384,0.0019573333362738294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,12288,0.0004992000137766202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,4096,0.0019434666881958645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,10240,0.0004970666641990344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,12288,0.0019456000377734502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,8192,0.00048533333465456964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,10240,0.001911466692884763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,8192,0.001874133323629697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,7168,0.0004917333212991555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,6144,0.0005226666728655497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,7168,0.0018965333700180053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,2560,0.0019551999866962433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,5120,0.00047786667322119073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,6144,0.0019477333873510361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,5120,0.001937066639463107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,3584,0.0004906666775544484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,1536,0.001924266666173935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,4096,0.001874133323629697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,3072,0.0004949333456655343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,3072,0.001966933285196622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,2560,0.0019466667125622433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,2048,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,2048,0.0018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,256,0.0018911999960740406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,1536,0.000516266655176878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,1024,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,1536,0.001926400015751521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,1024,0.001924266666173935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,768,0.0004906666775544484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,512,0.0005098666685322921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,768,0.0019018666197856266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,256,0.0005056000004212062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,512,0.0019253333409627277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,128,0.0005119999870657921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,256,0.0018751999984184902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,64,0.0005557333429654439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,32,0.0004906666775544484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,65536,0.0004821333413322766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,51200,0.0004992000137766202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,65536,0.002005333329240481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,16384,0.0005034666818877061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,51200,0.0019253333409627277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,12288,0.0004927999960879485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,16384,0.0019274666905403138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,2048,6144,0.001911466692884763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,10240,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,12288,0.0019690666347742082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,8192,0.0005450666571656863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,10240,0.0018357332795858384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,7168,0.000596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,8192,0.0019914666811625165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,6144,0.0005898666878541311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,7168,0.0019007999449968337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,5120,0.0005397333453098933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,6144,0.0019765333582957584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,4096,0.0005034666818877061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,5120,0.0020128000527620316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,3584,0.0005365333209435146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,4096,0.001934933289885521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,3072,0.0005258666972319285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,3584,0.001956266661485036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,2560,0.0005439999823768934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,3072,0.0019136000424623489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,2048,0.0005034666818877061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,2560,0.002004266654451688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,4096,0.000510933343321085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,1536,0.0005002666575213273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,2048,0.001963733385006587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,1024,0.0004992000137766202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,1536,0.0018634666999181111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,768,0.0005439999823768934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,3584,0.0018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1536,2560,0.000508799993743499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,512,0.0004896000027656555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,768,0.0019413333386182785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,256,0.0004981333389878273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,512,0.0018794666975736618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,128,0.000549333356320858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,256,0.0018709332992633183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,64,0.0005215999980767567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,128,0.0019167999426523846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,1024,32,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,51200,0.0005034666818877061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,65536,0.0005354666461547216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,16384,0.00047573332364360496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,65536,0.0018922666708628337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,51200,0.0019082666685183841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,12288,0.0004842666598657767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,16384,0.0019445333629846573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,12288,0.0019733333339293797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,10240,0.0019391999890406926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,8192,0.00046613334367672605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,7168,0.0005653333539764086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,8192,0.0019007999449968337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,6144,0.0004949333456655343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,7168,0.0019381333142518998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,5120,0.00048533333465456964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,6144,0.0019648000597953796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,4096,0.0004981333389878273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,3584,0.0004959999894102415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,3584,0.0019317333896954852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,3072,0.0005013333323101203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1536,128,0.0018933333456516265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,2560,0.0005034666818877061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,2560,0.0019338666150967281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,2048,0.0004896000027656555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,2048,0.0019530666371186574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,1536,0.000516266655176878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,1536,0.0019296000401178997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,1024,0.0005045333256324132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,1024,0.0018581333259741466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,768,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,768,0.001876266673207283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,512,0.0019061333189407986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,256,0.000514133336643378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,256,0.0019626667102177935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,128,0.0004959999894102415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,1024,1024,0.0018954666952292125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,128,0.001970133309563001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,64,0.0005013333323101203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,32,0.000514133336643378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,65536,0.0004981333389878273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,51200,0.000506666675209999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,65536,0.0019082666685183841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,16384,0.0004885333279768627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,51200,0.0018538666268189748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,12288,0.0004927999960879485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,10240,0.0005258666972319285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,12288,0.0019285333653291064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,10240,0.0005312000090877216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,5120,0.0022528000175952912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,10240,0.0019445333629846573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,8192,0.0004906666775544484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,7168,0.0005408000200986862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,7168,0.0020128000527620316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,6144,0.0005333333586653073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,3072,0.0018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,5120,0.0005568000177542369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,5120,0.0021087999145189924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,4096,0.0005674666414658229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,4096,0.0019285333653291064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,3584,0.0005429333075881004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,3584,0.0019125333676735562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,3072,0.0005514666438102723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,3072,0.0019071999937295913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,768,512,0.0005386666705211004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,2560,0.0019445333629846573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,2048,0.0005013333323101203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,2048,0.002160000056028366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,1536,0.0005226666728655497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,1536,0.001970133309563001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,1024,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,1024,0.0019274666905403138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,768,0.0005290666595101357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,16384,0.0018901333212852478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,768,0.0018965333700180053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,512,0.0005077333189547062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,256,0.0004970666641990344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,256,0.0018997333943843841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,128,0.0004885333279768627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,768,4096,0.0019466667125622433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,64,0.0005258666972319285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,128,0.001985066632429759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,8192,0.0018773333479960759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,32,0.0005098666685322921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,65536,0.0005119999870657921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,51200,0.0004767999984323978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,65536,0.0019018666197856266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,16384,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,51200,0.0019530666371186574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,12288,0.0004992000137766202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,10240,0.00048746665318806964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,12288,0.001956266661485036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,6144,0.00186666672428449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,8192,0.000510933343321085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,10240,0.0019359999646743138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,7168,0.0004949333456655343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,8192,0.0018794666975736618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,6144,0.0004992000137766202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,7168,0.0019317333896954852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,6144,0.0019413333386182785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,5120,0.00048640000944336255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,4096,0.000513066661854585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,5120,0.002001066630085309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,3584,0.0005077333189547062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,4096,0.001983999957640966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,3072,0.0004906666775544484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,3584,0.0020117333779732386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,2560,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,3072,0.0019221333165963489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,2048,0.0005002666575213273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,2048,0.0018826667219400406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,1536,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,1024,0.0004970666641990344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,1536,0.0018816000471512477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,768,0.0005194666795432567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,768,0.001937066639463107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,512,2560,0.000549333356320858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,512,0.0007402666533986728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,256,0.0005578666925430298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,512,0.0019882666567961377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,128,0.0006154666965206463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,128,0.0019946667055288953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,64,0.0005312000090877216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,256,32,0.0005813333516319593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,65536,0.0007327999919652939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,65536,0.002002133304874102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,51200,0.0005461333319544792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,51200,0.001971199984351794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,16384,0.0005290666595101357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,512,512,0.001961600035429001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,12288,0.0004842666598657767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,16384,0.0019648000597953796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,10240,0.0005322666838765144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,12288,0.0019082666685183841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,10240,0.0018858666221300761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,8192,0.0005333333586653073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,7168,0.0004927999960879485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,8192,0.0019413333386182785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,6144,0.0005045333256324132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,7168,0.002002133304874102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,16384,0.001874133323629697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,5120,0.0005215999980767567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,6144,0.0019776000330845515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,4096,0.000514133336643378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,5120,0.0019477333873510361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,3584,0.0007424000029762586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,4096,0.0019445333629846573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,3584,0.001975466683506966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,3072,0.00048533333465456964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,2560,0.0005152000114321709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,3072,0.0019338666150967281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,2048,0.0005248000224431355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,2560,0.001911466692884763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,1536,0.0004927999960879485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,2048,0.0020234666764736174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,1024,0.0007509333391984304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,768,0.0004970666641990344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,1024,0.0019861333072185515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,512,0.0005205333232879639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,768,0.0019317333896954852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,256,0.0004970666641990344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,512,0.0019541333119074506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,128,0.0005119999870657921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,256,0.001876266673207283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,64,0.0005610666548212369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,1024,0.0019210666418075562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,128,32,0.0005802666768431663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,65536,0.0004959999894102415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,51200,0.0004906666775544484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,16384,0.0005184000047544639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,12288,0.0005450666571656863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,8192,0.0004906666775544484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,7168,0.0004992000137766202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,256,0.0019765333582957584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,6144,0.0005152000114321709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,5120,0.0006464000170429547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,3584,0.0005013333323101203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,3072,0.0005418666948874792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,2560,0.0005077333189547062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,2048,0.0005002666575213273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,1536,0.0004906666775544484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,1024,0.0005226666728655497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,768,0.000516266655176878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,512,0.0005045333256324132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,256,0.0004938666708767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,128,0.0005077333189547062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,64,0.0005013333323101203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,32,0.0004981333389878273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,65536,0.0004906666775544484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,51200,0.000508799993743499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,16384,0.0005045333256324132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,12288,0.000514133336643378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,10240,0.0004970666641990344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,8192,0.0005056000004212062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,1536,0.001934933289885521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,7168,0.0005024000070989132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,6144,0.0005215999980767567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,256,2560,0.0019648000597953796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,5120,0.0005077333189547062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,4096,0.0005375999957323074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,3584,0.000513066661854585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,3072,0.0005717333406209946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,2048,0.0005439999823768934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,1536,0.0005535999933878581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,1024,0.0005685333162546158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,768,0.0004981333389878273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,10240,0.0004992000137766202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,512,0.0005535999933878581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,256,0.000513066661854585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,128,0.0005621333296100298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,64,0.000510933343321085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,32,0.000510933343321085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,64,4096,0.0005002666575213273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,0,32,2560,0.0006453333422541619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,0,128,128,0.001865600049495697
